stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8282072: G1: Rename CardSetPtr to CardSetContainerPtr

Browse Source 
Reviewed-by: ayang, tschatzl
This commit is contained in:
Ivan Walulya 2022-03-17 08:54:29 +00:00
parent 3da5204b3c
commit a5ebcc0c04
8 changed files with 346 additions and 353 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

405
src/hotspot/share/gc/g1/g1CardSet.cpp
View File

@ -26,25 +26,19 @@
#include "gc/g1/g1CardSet.inline.hpp"
#include "gc/g1/g1CardSetContainers.inline.hpp"
#include "gc/g1/g1CardSetMemory.inline.hpp"
#include "gc/g1/g1FromCardCache.hpp"
#include "gc/g1/heapRegion.inline.hpp"
#include "gc/shared/gcLogPrecious.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "memory/allocation.inline.hpp"
#include "runtime/atomic.hpp"
#include "runtime/globals_extension.hpp"
#include "runtime/mutex.hpp"
#include "utilities/bitMap.inline.hpp"
#include "utilities/concurrentHashTable.inline.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/lockFreeStack.hpp"
#include "utilities/spinYield.hpp"
#include "gc/shared/gcLogPrecious.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "runtime/java.hpp"
G1CardSet::ContainerPtr G1CardSet::FullCardSet = (G1CardSet::ContainerPtr)-1;
G1CardSet::CardSetPtr G1CardSet::FullCardSet = (G1CardSet::CardSetPtr)-1;
static uint default_log2_card_region_per_region() {
static uint default_log2_card_regions_per_region() {
uint log2_card_regions_per_heap_region = 0;
const uint card_container_limit = G1CardSetContainer::LogCardsPerRegionLimit;
@ -62,7 +56,7 @@ G1CardSetConfiguration::G1CardSetConfiguration() :
G1RemSetHowlNumBuckets, /* num_buckets_in_howl */
(double)G1RemSetCoarsenHowlToFullPercent / 100, /* cards_in_howl_threshold_percent */
(uint)HeapRegion::CardsPerRegion, /* max_cards_in_cardset */
default_log2_card_region_per_region()) /* log2_card_region_per_region */
default_log2_card_regions_per_region()) /* log2_card_regions_per_region */
{
assert((_log2_card_regions_per_heap_region + _log2_cards_per_card_region) == (uint)HeapRegion::LogCardsPerRegion,
"inconsistent heap region virtualization setup");
@ -73,7 +67,7 @@ G1CardSetConfiguration::G1CardSetConfiguration(uint max_cards_in_array,
uint max_buckets_in_howl,
double cards_in_howl_threshold_percent,
uint max_cards_in_card_set,
uint log2_card_region_per_region) :
uint log2_card_regions_per_region) :
G1CardSetConfiguration(log2i_exact(max_cards_in_card_set), /* inline_ptr_bits_per_card */
max_cards_in_array, /* max_cards_in_array */
cards_in_bitmap_threshold_percent, /* cards_in_bitmap_threshold_percent */
@ -82,7 +76,7 @@ G1CardSetConfiguration::G1CardSetConfiguration(uint max_cards_in_array,
max_buckets_in_howl),
cards_in_howl_threshold_percent, /* cards_in_howl_threshold_percent */
max_cards_in_card_set, /* max_cards_in_cardset */
log2_card_region_per_region)
log2_card_regions_per_region)
{ }
G1CardSetConfiguration::G1CardSetConfiguration(uint inline_ptr_bits_per_card,
@ -197,7 +191,7 @@ void G1CardSetCoarsenStats::print_on(outputStream* out) {
}
class G1CardSetHashTable : public CHeapObj<mtGCCardSet> {
using CardSetPtr = G1CardSet::CardSetPtr;
using ContainerPtr = G1CardSet::ContainerPtr;
// Did we insert at least one card in the table?
bool volatile _inserted_card;
@ -231,12 +225,12 @@ class G1CardSetHashTable : public CHeapObj<mtGCCardSet> {
};
class G1CardSetHashTableScan : public StackObj {
G1CardSet::CardSetPtrClosure* _scan_f;
G1CardSet::ContainerPtrClosure* _scan_f;
public:
explicit G1CardSetHashTableScan(G1CardSet::CardSetPtrClosure* f) : _scan_f(f) { }
explicit G1CardSetHashTableScan(G1CardSet::ContainerPtrClosure* f) : _scan_f(f) { }
bool operator()(G1CardSetHashTableValue* value) {
_scan_f->do_cardsetptr(value->_region_idx, value->_num_occupied, value->_card_set);
_scan_f->do_containerptr(value->_region_idx, value->_num_occupied, value->_container);
return true;
}
};
@ -284,19 +278,19 @@ public:
return found.value();
}
void iterate_safepoint(G1CardSet::CardSetPtrClosure* cl2) {
void iterate_safepoint(G1CardSet::ContainerPtrClosure* cl2) {
G1CardSetHashTableScan cl(cl2);
_table.do_safepoint_scan(cl);
}
void iterate(G1CardSet::CardSetPtrClosure* cl2) {
void iterate(G1CardSet::ContainerPtrClosure* cl2) {
G1CardSetHashTableScan cl(cl2);
_table.do_scan(Thread::current(), cl);
}
void reset() {
if (Atomic::load(&_inserted_card)) {
_table.unsafe_reset(InitialLogTableSize);
_table.unsafe_reset(InitialLogTableSize);
Atomic::store(&_inserted_card, false);
}
}
@ -343,93 +337,93 @@ G1CardSet::~G1CardSet() {
_mm->flush();
}
uint G1CardSet::card_set_type_to_mem_object_type(uintptr_t type) const {
assert(type == G1CardSet::CardSetArrayOfCards ||
type == G1CardSet::CardSetBitMap ||
type == G1CardSet::CardSetHowl, "should not allocate card set type %zu", type);
uint G1CardSet::container_type_to_mem_object_type(uintptr_t type) const {
assert(type == G1CardSet::ContainerArrayOfCards ||
type == G1CardSet::ContainerBitMap ||
type == G1CardSet::ContainerHowl, "should not allocate container type %zu", type);
return (uint)type;
}
uint8_t* G1CardSet::allocate_mem_object(uintptr_t type) {
return _mm->allocate(card_set_type_to_mem_object_type(type));
return _mm->allocate(container_type_to_mem_object_type(type));
}
void G1CardSet::free_mem_object(CardSetPtr card_set) {
assert(card_set != G1CardSet::FreeCardSet, "should not free Free card set");
assert(card_set != G1CardSet::FullCardSet, "should not free Full card set");
void G1CardSet::free_mem_object(ContainerPtr container) {
assert(container != G1CardSet::FreeCardSet, "should not free container FreeCardSet");
assert(container != G1CardSet::FullCardSet, "should not free container FullCardSet");
uintptr_t type = card_set_type(card_set);
void* value = strip_card_set_type(card_set);
uintptr_t type = container_type(container);
void* value = strip_container_type(container);
assert(type == G1CardSet::CardSetArrayOfCards ||
type == G1CardSet::CardSetBitMap ||
type == G1CardSet::CardSetHowl, "should not free card set type %zu", type);
assert(type == G1CardSet::ContainerArrayOfCards ||
type == G1CardSet::ContainerBitMap ||
type == G1CardSet::ContainerHowl, "should not free card set type %zu", type);
assert(static_cast<G1CardSetContainer*>(value)->refcount() == 1, "must be");
_mm->free(card_set_type_to_mem_object_type(type), value);
_mm->free(container_type_to_mem_object_type(type), value);
}
G1CardSet::CardSetPtr G1CardSet::acquire_card_set(CardSetPtr volatile* card_set_addr) {
G1CardSet::ContainerPtr G1CardSet::acquire_container(ContainerPtr volatile* container_addr) {
// Update reference counts under RCU critical section to avoid a
// use-after-cleapup bug where we increment a reference count for
// an object whose memory has already been cleaned up and reused.
GlobalCounter::CriticalSection cs(Thread::current());
while (true) {
// Get cardsetptr and increment refcount atomically wrt to memory reuse.
CardSetPtr card_set = Atomic::load_acquire(card_set_addr);
uint cs_type = card_set_type(card_set);
if (card_set == FullCardSet || cs_type == CardSetInlinePtr) {
return card_set;
// Get ContainerPtr and increment refcount atomically wrt to memory reuse.
ContainerPtr container = Atomic::load_acquire(container_addr);
uint cs_type = container_type(container);
if (container == FullCardSet || cs_type == ContainerInlinePtr) {
return container;
}
G1CardSetContainer* card_set_on_heap = (G1CardSetContainer*)strip_card_set_type(card_set);
G1CardSetContainer* container_on_heap = (G1CardSetContainer*)strip_container_type(container);
if (card_set_on_heap->try_increment_refcount()) {
assert(card_set_on_heap->refcount() >= 3, "Smallest value is 3");
return card_set;
if (container_on_heap->try_increment_refcount()) {
assert(container_on_heap->refcount() >= 3, "Smallest value is 3");
return container;
}
}
}
bool G1CardSet::release_card_set(CardSetPtr card_set) {
uint cs_type = card_set_type(card_set);
if (card_set == FullCardSet || cs_type == CardSetInlinePtr) {
bool G1CardSet::release_container(ContainerPtr container) {
uint cs_type = container_type(container);
if (container == FullCardSet || cs_type == ContainerInlinePtr) {
return false;
}
G1CardSetContainer* card_set_on_heap = (G1CardSetContainer*)strip_card_set_type(card_set);
return card_set_on_heap->decrement_refcount() == 1;
G1CardSetContainer* container_on_heap = (G1CardSetContainer*)strip_container_type(container);
return container_on_heap->decrement_refcount() == 1;
}
void G1CardSet::release_and_maybe_free_card_set(CardSetPtr card_set) {
if (release_card_set(card_set)) {
free_mem_object(card_set);
void G1CardSet::release_and_maybe_free_container(ContainerPtr container) {
if (release_container(container)) {
free_mem_object(container);
}
}
void G1CardSet::release_and_must_free_card_set(CardSetPtr card_set) {
bool should_free = release_card_set(card_set);
void G1CardSet::release_and_must_free_container(ContainerPtr container) {
bool should_free = release_container(container);
assert(should_free, "should have been the only one having a reference");
free_mem_object(card_set);
free_mem_object(container);
}
class G1ReleaseCardsets : public StackObj {
G1CardSet* _card_set;
using CardSetPtr = G1CardSet::CardSetPtr;
using ContainerPtr = G1CardSet::ContainerPtr;
void coarsen_to_full(CardSetPtr* card_set_addr) {
void coarsen_to_full(ContainerPtr* container_addr) {
while (true) {
CardSetPtr cur_card_set = Atomic::load_acquire(card_set_addr);
uint cs_type = G1CardSet::card_set_type(cur_card_set);
if (cur_card_set == G1CardSet::FullCardSet) {
ContainerPtr cur_container = Atomic::load_acquire(container_addr);
uint cs_type = G1CardSet::container_type(cur_container);
if (cur_container == G1CardSet::FullCardSet) {
return;
}
CardSetPtr old_value = Atomic::cmpxchg(card_set_addr, cur_card_set, G1CardSet::FullCardSet);
ContainerPtr old_value = Atomic::cmpxchg(container_addr, cur_container, G1CardSet::FullCardSet);
if (old_value == cur_card_set) {
_card_set->release_and_maybe_free_card_set(cur_card_set);
if (old_value == cur_container) {
_card_set->release_and_maybe_free_container(cur_container);
return;
}
}
@ -438,51 +432,51 @@ class G1ReleaseCardsets : public StackObj {
public:
explicit G1ReleaseCardsets(G1CardSet* card_set) : _card_set(card_set) { }
void operator ()(CardSetPtr* card_set_addr) {
coarsen_to_full(card_set_addr);
void operator ()(ContainerPtr* container_addr) {
coarsen_to_full(container_addr);
}
};
G1AddCardResult G1CardSet::add_to_array(CardSetPtr card_set, uint card_in_region) {
G1CardSetArray* array = card_set_ptr<G1CardSetArray>(card_set);
G1AddCardResult G1CardSet::add_to_array(ContainerPtr container, uint card_in_region) {
G1CardSetArray* array = container_ptr<G1CardSetArray>(container);
return array->add(card_in_region);
}
G1AddCardResult G1CardSet::add_to_howl(CardSetPtr parent_card_set,
uint card_region,
uint card_in_region,
bool increment_total) {
G1CardSetHowl* howl = card_set_ptr<G1CardSetHowl>(parent_card_set);
G1AddCardResult G1CardSet::add_to_howl(ContainerPtr parent_container,
uint card_region,
uint card_in_region,
bool increment_total) {
G1CardSetHowl* howl = container_ptr<G1CardSetHowl>(parent_container);
G1AddCardResult add_result;
CardSetPtr to_transfer = nullptr;
CardSetPtr card_set;
ContainerPtr to_transfer = nullptr;
ContainerPtr container;
uint bucket = _config->howl_bucket_index(card_in_region);
volatile CardSetPtr* bucket_entry = howl->get_card_set_addr(bucket);
ContainerPtr volatile* bucket_entry = howl->get_container_addr(bucket);
while (true) {
if (Atomic::load(&howl->_num_entries) >= _config->cards_in_howl_threshold()) {
return Overflow;
}
card_set = acquire_card_set(bucket_entry);
add_result = add_to_card_set(bucket_entry, card_set, card_region, card_in_region);
container = acquire_container(bucket_entry);
add_result = add_to_container(bucket_entry, container, card_region, card_in_region);
if (add_result != Overflow) {
break;
}
// Card set has overflown. Coarsen or retry.
bool coarsened = coarsen_card_set(bucket_entry, card_set, card_in_region, true /* within_howl */);
_coarsen_stats.record_coarsening(card_set_type(card_set) + G1CardSetCoarsenStats::CoarsenHowlOffset, !coarsened);
// Card set container has overflown. Coarsen or retry.
bool coarsened = coarsen_container(bucket_entry, container, card_in_region, true /* within_howl */);
_coarsen_stats.record_coarsening(container_type(container) + G1CardSetCoarsenStats::CoarsenHowlOffset, !coarsened);
if (coarsened) {
// We have been the one coarsening this card set (and in the process added that card).
// We successful coarsened this card set container (and in the process added the card).
add_result = Added;
to_transfer = card_set;
to_transfer = container;
break;
}
// Somebody else beat us to coarsening. Retry.
release_and_maybe_free_card_set(card_set);
release_and_maybe_free_container(container);
}
if (increment_total && add_result == Added) {
@ -490,91 +484,91 @@ G1AddCardResult G1CardSet::add_to_howl(CardSetPtr parent_card_set,
}
if (to_transfer != nullptr) {
transfer_cards_in_howl(parent_card_set, to_transfer, card_region);
transfer_cards_in_howl(parent_container, to_transfer, card_region);
}
release_and_maybe_free_card_set(card_set);
release_and_maybe_free_container(container);
return add_result;
}
G1AddCardResult G1CardSet::add_to_bitmap(CardSetPtr card_set, uint card_in_region) {
G1CardSetBitMap* bitmap = card_set_ptr<G1CardSetBitMap>(card_set);
G1AddCardResult G1CardSet::add_to_bitmap(ContainerPtr container, uint card_in_region) {
G1CardSetBitMap* bitmap = container_ptr<G1CardSetBitMap>(container);
uint card_offset = _config->howl_bitmap_offset(card_in_region);
return bitmap->add(card_offset, _config->cards_in_howl_bitmap_threshold(), _config->max_cards_in_howl_bitmap());
}
G1AddCardResult G1CardSet::add_to_inline_ptr(CardSetPtr volatile* card_set_addr, CardSetPtr card_set, uint card_in_region) {
G1CardSetInlinePtr value(card_set_addr, card_set);
G1AddCardResult G1CardSet::add_to_inline_ptr(ContainerPtr volatile* container_addr, ContainerPtr container, uint card_in_region) {
G1CardSetInlinePtr value(container_addr, container);
return value.add(card_in_region, _config->inline_ptr_bits_per_card(), _config->max_cards_in_inline_ptr());
}
G1CardSet::CardSetPtr G1CardSet::create_coarsened_array_of_cards(uint card_in_region, bool within_howl) {
G1CardSet::ContainerPtr G1CardSet::create_coarsened_array_of_cards(uint card_in_region, bool within_howl) {
uint8_t* data = nullptr;
CardSetPtr new_card_set;
ContainerPtr new_container;
if (within_howl) {
uint const size_in_bits = _config->max_cards_in_howl_bitmap();
uint card_offset = _config->howl_bitmap_offset(card_in_region);
data = allocate_mem_object(CardSetBitMap);
new (data) G1CardSetBitMap(card_offset, size_in_bits);
new_card_set = make_card_set_ptr(data, CardSetBitMap);
uint container_offset = _config->howl_bitmap_offset(card_in_region);
data = allocate_mem_object(ContainerBitMap);
new (data) G1CardSetBitMap(container_offset, size_in_bits);
new_container = make_container_ptr(data, ContainerBitMap);
} else {
data = allocate_mem_object(CardSetHowl);
data = allocate_mem_object(ContainerHowl);
new (data) G1CardSetHowl(card_in_region, _config);
new_card_set = make_card_set_ptr(data, CardSetHowl);
new_container = make_container_ptr(data, ContainerHowl);
}
return new_card_set;
return new_container;
}
bool G1CardSet::coarsen_card_set(volatile CardSetPtr* card_set_addr,
CardSetPtr cur_card_set,
uint card_in_region,
bool within_howl) {
CardSetPtr new_card_set = nullptr;
bool G1CardSet::coarsen_container(ContainerPtr volatile* container_addr,
ContainerPtr cur_container,
uint card_in_region,
bool within_howl) {
ContainerPtr new_container = nullptr;
switch (card_set_type(cur_card_set)) {
case CardSetArrayOfCards : {
new_card_set = create_coarsened_array_of_cards(card_in_region, within_howl);
switch (container_type(cur_container)) {
case ContainerArrayOfCards: {
new_container = create_coarsened_array_of_cards(card_in_region, within_howl);
break;
}
case CardSetBitMap: {
new_card_set = FullCardSet;
case ContainerBitMap: {
new_container = FullCardSet;
break;
}
case CardSetInlinePtr: {
case ContainerInlinePtr: {
uint const size = _config->max_cards_in_array();
uint8_t* data = allocate_mem_object(CardSetArrayOfCards);
uint8_t* data = allocate_mem_object(ContainerArrayOfCards);
new (data) G1CardSetArray(card_in_region, size);
new_card_set = make_card_set_ptr(data, CardSetArrayOfCards);
new_container = make_container_ptr(data, ContainerArrayOfCards);
break;
}
case CardSetHowl: {
new_card_set = FullCardSet; // anything will do at this point.
case ContainerHowl: {
new_container = FullCardSet; // anything will do at this point.
break;
}
default:
ShouldNotReachHere();
}
CardSetPtr old_value = Atomic::cmpxchg(card_set_addr, cur_card_set, new_card_set); // Memory order?
if (old_value == cur_card_set) {
ContainerPtr old_value = Atomic::cmpxchg(container_addr, cur_container, new_container); // Memory order?
if (old_value == cur_container) {
// Success. Indicate that the cards from the current card set must be transferred
// by this caller.
// Release the hash table reference to the card. The caller still holds the
// reference to this card set, so it can never be released (and we do not need to
// check its result).
bool should_free = release_card_set(cur_card_set);
bool should_free = release_container(cur_container);
assert(!should_free, "must have had more than one reference");
// Free containers if cur_card_set is CardSetHowl
if (card_set_type(cur_card_set) == CardSetHowl) {
// Free containers if cur_container is ContainerHowl
if (container_type(cur_container) == ContainerHowl) {
G1ReleaseCardsets rel(this);
card_set_ptr<G1CardSetHowl>(cur_card_set)->iterate(rel, _config->num_buckets_in_howl());
container_ptr<G1CardSetHowl>(cur_container)->iterate(rel, _config->num_buckets_in_howl());
}
return true;
} else {
// Somebody else beat us to coarsening that card set. Exit, but clean up first.
if (new_card_set != FullCardSet) {
assert(new_card_set != nullptr, "must not be");
release_and_must_free_card_set(new_card_set);
if (new_container != FullCardSet) {
assert(new_container != nullptr, "must not be");
release_and_must_free_container(new_container);
}
return false;
}
@ -591,34 +585,34 @@ public:
}
};
void G1CardSet::transfer_cards(G1CardSetHashTableValue* table_entry, CardSetPtr source_card_set, uint card_region) {
assert(source_card_set != FullCardSet, "Should not need to transfer from full");
// Need to transfer old entries unless there is a Full card set in place now, i.e.
// the old type has been CardSetBitMap. "Full" contains all elements anyway.
if (card_set_type(source_card_set) != CardSetHowl) {
void G1CardSet::transfer_cards(G1CardSetHashTableValue* table_entry, ContainerPtr source_container, uint card_region) {
assert(source_container != FullCardSet, "Should not need to transfer from FullCardSet");
// Need to transfer old entries unless there is a Full card set container in place now, i.e.
// the old type has been ContainerBitMap. "Full" contains all elements anyway.
if (container_type(source_container) != ContainerHowl) {
G1TransferCard iter(this, card_region);
iterate_cards_during_transfer(source_card_set, iter);
iterate_cards_during_transfer(source_container, iter);
} else {
assert(card_set_type(source_card_set) == CardSetHowl, "must be");
assert(container_type(source_container) == ContainerHowl, "must be");
// Need to correct for that the Full remembered set occupies more cards than the
// AoCS before.
Atomic::add(&_num_occupied, _config->max_cards_in_region() - table_entry->_num_occupied, memory_order_relaxed);
}
}
void G1CardSet::transfer_cards_in_howl(CardSetPtr parent_card_set,
CardSetPtr source_card_set,
uint card_region) {
assert(card_set_type(parent_card_set) == CardSetHowl, "must be");
assert(source_card_set != FullCardSet, "Should not need to transfer from full");
void G1CardSet::transfer_cards_in_howl(ContainerPtr parent_container,
ContainerPtr source_container,
uint card_region) {
assert(container_type(parent_container) == ContainerHowl, "must be");
assert(source_container != FullCardSet, "Should not need to transfer from full");
// Need to transfer old entries unless there is a Full card set in place now, i.e.
// the old type has been CardSetBitMap.
if (card_set_type(source_card_set) != CardSetBitMap) {
// We only need to transfer from anything below CardSetBitMap.
// the old type has been ContainerBitMap.
if (container_type(source_container) != ContainerBitMap) {
// We only need to transfer from anything below ContainerBitMap.
G1TransferCard iter(this, card_region);
iterate_cards_during_transfer(source_card_set, iter);
iterate_cards_during_transfer(source_container, iter);
} else {
uint diff = _config->max_cards_in_howl_bitmap() - card_set_ptr<G1CardSetBitMap>(source_card_set)->num_bits_set();
uint diff = _config->max_cards_in_howl_bitmap() - container_ptr<G1CardSetBitMap>(source_container)->num_bits_set();
// Need to correct for that the Full remembered set occupies more cards than the
// bitmap before.
@ -627,10 +621,10 @@ void G1CardSet::transfer_cards_in_howl(CardSetPtr parent_card_set,
// G1CardSet::add_to_howl after coarsening.
diff -= 1;
G1CardSetHowl* howling_array = card_set_ptr<G1CardSetHowl>(parent_card_set);
G1CardSetHowl* howling_array = container_ptr<G1CardSetHowl>(parent_container);
Atomic::add(&howling_array->_num_entries, diff, memory_order_relaxed);
G1CardSetHashTableValue* table_entry = get_card_set(card_region);
G1CardSetHashTableValue* table_entry = get_container(card_region);
assert(table_entry != nullptr, "Table entry not found for transferred cards");
Atomic::add(&table_entry->_num_occupied, diff, memory_order_relaxed);
@ -639,72 +633,75 @@ void G1CardSet::transfer_cards_in_howl(CardSetPtr parent_card_set,
}
}
G1AddCardResult G1CardSet::add_to_card_set(volatile CardSetPtr* card_set_addr, CardSetPtr card_set, uint card_region, uint card_in_region, bool increment_total) {
assert(card_set_addr != nullptr, "Cannot add to empty cardset");
G1AddCardResult G1CardSet::add_to_container(ContainerPtr volatile* container_addr,
ContainerPtr container,
uint card_region,
uint card_in_region,
bool increment_total) {
assert(container_addr != nullptr, "must be");
G1AddCardResult add_result;
switch (card_set_type(card_set)) {
case CardSetInlinePtr: {
add_result = add_to_inline_ptr(card_set_addr, card_set, card_in_region);
switch (container_type(container)) {
case ContainerInlinePtr: {
add_result = add_to_inline_ptr(container_addr, container, card_in_region);
break;
}
case CardSetArrayOfCards : {
add_result = add_to_array(card_set, card_in_region);
case ContainerArrayOfCards: {
add_result = add_to_array(container, card_in_region);
break;
}
case CardSetBitMap: {
add_result = add_to_bitmap(card_set, card_in_region);
case ContainerBitMap: {
add_result = add_to_bitmap(container, card_in_region);
break;
}
case CardSetHowl: {
assert(CardSetHowl == card_set_type(FullCardSet), "must be");
if (card_set == FullCardSet) {
case ContainerHowl: {
assert(ContainerHowl == container_type(FullCardSet), "must be");
if (container == FullCardSet) {
return Found;
}
add_result = add_to_howl(card_set, card_region, card_in_region, increment_total);
add_result = add_to_howl(container, card_region, card_in_region, increment_total);
break;
}
default:
ShouldNotReachHere();
}
return add_result;
}
G1CardSetHashTableValue* G1CardSet::get_or_add_card_set(uint card_region, bool* should_grow_table) {
G1CardSetHashTableValue* G1CardSet::get_or_add_container(uint card_region, bool* should_grow_table) {
return _table->get_or_add(card_region, should_grow_table);
}
G1CardSetHashTableValue* G1CardSet::get_card_set(uint card_region) {
G1CardSetHashTableValue* G1CardSet::get_container(uint card_region) {
return _table->get(card_region);
}
G1AddCardResult G1CardSet::add_card(uint card_region, uint card_in_region, bool increment_total) {
G1AddCardResult add_result;
CardSetPtr to_transfer = nullptr;
CardSetPtr card_set;
ContainerPtr to_transfer = nullptr;
ContainerPtr container;
bool should_grow_table = false;
G1CardSetHashTableValue* table_entry = get_or_add_card_set(card_region, &should_grow_table);
G1CardSetHashTableValue* table_entry = get_or_add_container(card_region, &should_grow_table);
while (true) {
card_set = acquire_card_set(&table_entry->_card_set);
add_result = add_to_card_set(&table_entry->_card_set, card_set, card_region, card_in_region, increment_total);
container = acquire_container(&table_entry->_container);
add_result = add_to_container(&table_entry->_container, container, card_region, card_in_region, increment_total);
if (add_result != Overflow) {
break;
}
// Card set has overflown. Coarsen or retry.
bool coarsened = coarsen_card_set(&table_entry->_card_set, card_set, card_in_region);
_coarsen_stats.record_coarsening(card_set_type(card_set), !coarsened);
bool coarsened = coarsen_container(&table_entry->_container, container, card_in_region);
_coarsen_stats.record_coarsening(container_type(container), !coarsened);
if (coarsened) {
// We have been the one coarsening this card set (and in the process added that card).
// We successful coarsened this card set container (and in the process added the card).
add_result = Added;
to_transfer = card_set;
to_transfer = container;
break;
}
// Somebody else beat us to coarsening. Retry.
release_and_maybe_free_card_set(card_set);
release_and_maybe_free_container(container);
}
if (increment_total && add_result == Added) {
@ -718,7 +715,7 @@ G1AddCardResult G1CardSet::add_card(uint card_region, uint card_in_region, bool
transfer_cards(table_entry, to_transfer, card_region);
}
release_and_maybe_free_card_set(card_set);
release_and_maybe_free_container(container);
return add_result;
}
@ -727,29 +724,29 @@ bool G1CardSet::contains_card(uint card_region, uint card_in_region) {
assert(card_in_region < _config->max_cards_in_region(),
"Card %u is beyond max %u", card_in_region, _config->max_cards_in_region());
// Protect the card set from reclamation.
// Protect the card set container from reclamation.
GlobalCounter::CriticalSection cs(Thread::current());
G1CardSetHashTableValue* table_entry = get_card_set(card_region);
G1CardSetHashTableValue* table_entry = get_container(card_region);
if (table_entry == nullptr) {
return false;
}
CardSetPtr card_set = table_entry->_card_set;
if (card_set == FullCardSet) {
ContainerPtr container = table_entry->_container;
if (container == FullCardSet) {
// contains_card() is not a performance critical method so we do not hide that
// case in the switch below.
return true;
}
switch (card_set_type(card_set)) {
case CardSetInlinePtr: {
G1CardSetInlinePtr ptr(card_set);
switch (container_type(container)) {
case ContainerInlinePtr: {
G1CardSetInlinePtr ptr(container);
return ptr.contains(card_in_region, _config->inline_ptr_bits_per_card());
}
case CardSetArrayOfCards : return card_set_ptr<G1CardSetArray>(card_set)->contains(card_in_region);
case CardSetBitMap: return card_set_ptr<G1CardSetBitMap>(card_set)->contains(card_in_region, _config->max_cards_in_howl_bitmap());
case CardSetHowl: {
G1CardSetHowl* howling_array = card_set_ptr<G1CardSetHowl>(card_set);
case ContainerArrayOfCards: return container_ptr<G1CardSetArray>(container)->contains(card_in_region);
case ContainerBitMap: return container_ptr<G1CardSetBitMap>(container)->contains(card_in_region, _config->max_cards_in_howl_bitmap());
case ContainerHowl: {
G1CardSetHowl* howling_array = container_ptr<G1CardSetHowl>(container);
return howling_array->contains(card_in_region, _config);
}
@ -759,53 +756,53 @@ bool G1CardSet::contains_card(uint card_region, uint card_in_region) {
}
void G1CardSet::print_info(outputStream* st, uint card_region, uint card_in_region) {
G1CardSetHashTableValue* table_entry = get_card_set(card_region);
G1CardSetHashTableValue* table_entry = get_container(card_region);
if (table_entry == nullptr) {
st->print("NULL card set");
return;
}
CardSetPtr card_set = table_entry->_card_set;
if (card_set == FullCardSet) {
ContainerPtr container = table_entry->_container;
if (container == FullCardSet) {
st->print("FULL card set)");
return;
}
switch (card_set_type(card_set)) {
case CardSetInlinePtr: {
switch (container_type(container)) {
case ContainerInlinePtr: {
st->print("InlinePtr not containing %u", card_in_region);
break;
}
case CardSetArrayOfCards : {
case ContainerArrayOfCards: {
st->print("AoC not containing %u", card_in_region);
break;
}
case CardSetBitMap: {
case ContainerBitMap: {
st->print("BitMap not containing %u", card_in_region);
break;
}
case CardSetHowl: {
st->print("CardSetHowl not containing %u", card_in_region);
case ContainerHowl: {
st->print("ContainerHowl not containing %u", card_in_region);
break;
}
default: st->print("Unknown card set type %u", card_set_type(card_set)); ShouldNotReachHere(); break;
default: st->print("Unknown card set container type %u", container_type(container)); ShouldNotReachHere(); break;
}
}
template <class CardVisitor>
void G1CardSet::iterate_cards_during_transfer(CardSetPtr const card_set, CardVisitor& cl) {
uint type = card_set_type(card_set);
assert(type == CardSetInlinePtr || type == CardSetArrayOfCards,
void G1CardSet::iterate_cards_during_transfer(ContainerPtr const container, CardVisitor& cl) {
uint type = container_type(container);
assert(type == ContainerInlinePtr || type == ContainerArrayOfCards,
"invalid card set type %d to transfer from",
card_set_type(card_set));
container_type(container));
switch (type) {
case CardSetInlinePtr: {
G1CardSetInlinePtr ptr(card_set);
case ContainerInlinePtr: {
G1CardSetInlinePtr ptr(container);
ptr.iterate(cl, _config->inline_ptr_bits_per_card());
return;
}
case CardSetArrayOfCards : {
card_set_ptr<G1CardSetArray>(card_set)->iterate(cl);
case ContainerArrayOfCards: {
container_ptr<G1CardSetArray>(container)->iterate(cl);
return;
}
default:
@ -813,7 +810,7 @@ void G1CardSet::iterate_cards_during_transfer(CardSetPtr const card_set, CardVis
}
}
void G1CardSet::iterate_containers(CardSetPtrClosure* cl, bool at_safepoint) {
void G1CardSet::iterate_containers(ContainerPtrClosure* cl, bool at_safepoint) {
if (at_safepoint) {
_table->iterate_safepoint(cl);
} else {
@ -844,7 +841,7 @@ public:
};
template <typename Closure, template <typename> class CardOrRanges>
class G1CardSetContainersClosure : public G1CardSet::CardSetPtrClosure {
class G1CardSetContainersClosure : public G1CardSet::ContainerPtrClosure {
G1CardSet* _card_set;
Closure& _cl;
@ -855,9 +852,9 @@ public:
_card_set(card_set),
_cl(cl) { }
void do_cardsetptr(uint region_idx, size_t num_occupied, G1CardSet::CardSetPtr card_set) override {
void do_containerptr(uint region_idx, size_t num_occupied, G1CardSet::ContainerPtr container) override {
CardOrRanges<Closure> cl(_cl, region_idx);
_card_set->iterate_cards_or_ranges_in_container(card_set, cl);
_card_set->iterate_cards_or_ranges_in_container(container, cl);
}
};
@ -879,13 +876,13 @@ size_t G1CardSet::occupied() const {
}
size_t G1CardSet::num_containers() {
class GetNumberOfContainers : public CardSetPtrClosure {
class GetNumberOfContainers : public ContainerPtrClosure {
public:
size_t _count;
GetNumberOfContainers() : CardSetPtrClosure(), _count(0) { }
GetNumberOfContainers() : ContainerPtrClosure(), _count(0) { }
void do_cardsetptr(uint region_idx, size_t num_occupied, CardSetPtr card_set) override {
void do_containerptr(uint region_idx, size_t num_occupied, ContainerPtr container) override {
_count++;
}
} cl;

133
src/hotspot/share/gc/g1/g1CardSet.hpp
View File

@ -26,10 +26,7 @@
#define SHARE_GC_G1_G1CARDSET_HPP
#include "memory/allocation.hpp"
#include "memory/padded.hpp"
#include "oops/oopsHierarchy.hpp"
#include "utilities/concurrentHashTable.hpp"
#include "utilities/lockFreeStack.hpp"
class G1CardSetAllocOptions;
class G1CardSetHashTable;
@ -147,10 +144,10 @@ public:
class G1CardSetCoarsenStats {
public:
// Number of entries in the statistics tables: since we index with the source
// cardset of the coarsening, this is the total number of combinations of
// card sets - 1.
// container of the coarsening, this is the total number of combinations of
// card set containers - 1.
static constexpr size_t NumCoarsenCategories = 7;
// Coarsening statistics for the possible CardSetPtr in the Howl card set
// Coarsening statistics for the possible ContainerPtr in the Howl card set
// start from this offset.
static constexpr size_t CoarsenHowlOffset = 4;
@ -173,14 +170,14 @@ public:
void print_on(outputStream* out);
};
// Sparse set of card indexes comprising a remembered set on the Java heap. Card
// Set of card indexes comprising a remembered set on the Java heap. Card
// size is assumed to be card table card size.
//
// Technically it is implemented using a ConcurrentHashTable that stores a card
// set container for every region containing at least one card.
//
// There are in total five different containers, encoded in the ConcurrentHashTable
// node as CardSetPtr. A CardSetPtr may cover the whole region or just a part of
// node as ContainerPtr. A ContainerPtr may cover the whole region or just a part of
// it.
// See its description below for more information.
class G1CardSet : public CHeapObj<mtGCCardSet> {
@ -194,46 +191,46 @@ class G1CardSet : public CHeapObj<mtGCCardSet> {
static G1CardSetCoarsenStats _coarsen_stats; // Coarsening statistics since VM start.
static G1CardSetCoarsenStats _last_coarsen_stats; // Coarsening statistics at last GC.
public:
// Two lower bits are used to encode the card storage types
static const uintptr_t CardSetPtrHeaderSize = 2;
// Two lower bits are used to encode the card set container types
static const uintptr_t ContainerPtrHeaderSize = 2;
// CardSetPtr represents the card storage type of a given covered area. It encodes
// a type in the LSBs, in addition to having a few significant values.
// ContainerPtr represents the card set container type of a given covered area.
// It encodes a type in the LSBs, in addition to having a few significant values.
//
// Possible encodings:
//
// 0...00000 free (Empty, should never happen)
// 1...11111 full All card indexes in the whole area this CardSetPtr covers are part of this container.
// X...XXX00 inline-ptr-cards A handful of card indexes covered by this CardSetPtr are encoded within the CardSetPtr.
// 1...11111 full All card indexes in the whole area this ContainerPtr covers are part of this container.
// X...XXX00 inline-ptr-cards A handful of card indexes covered by this ContainerPtr are encoded within the ContainerPtr.
// X...XXX01 array of cards The container is a contiguous array of card indexes.
// X...XXX10 bitmap The container uses a bitmap to determine whether a given index is part of this set.
// X...XXX11 howl This is a card set container containing an array of CardSetPtr, with each CardSetPtr
// X...XXX11 howl This is a card set container containing an array of ContainerPtr, with each ContainerPtr
// limited to a sub-range of the original range. Currently only one level of this
// container is supported.
typedef void* CardSetPtr;
using ContainerPtr = void*;
// Coarsening happens in the order below:
// CardSetInlinePtr -> CardSetArrayOfCards -> CardSetHowl -> Full
// Corsening of containers inside the CardSetHowl happens in the order:
// CardSetInlinePtr -> CardSetArrayOfCards -> CardSetBitMap -> Full
static const uintptr_t CardSetInlinePtr = 0x0;
static const uintptr_t CardSetArrayOfCards = 0x1;
static const uintptr_t CardSetBitMap = 0x2;
static const uintptr_t CardSetHowl = 0x3;
// ContainerInlinePtr -> ContainerArrayOfCards -> ContainerHowl -> Full
// Corsening of containers inside the ContainerHowl happens in the order:
// ContainerInlinePtr -> ContainerArrayOfCards -> ContainerBitMap -> Full
static const uintptr_t ContainerInlinePtr = 0x0;
static const uintptr_t ContainerArrayOfCards = 0x1;
static const uintptr_t ContainerBitMap = 0x2;
static const uintptr_t ContainerHowl = 0x3;
// The special sentinel values
static constexpr CardSetPtr FreeCardSet = nullptr;
// Unfortunately we can't make (G1CardSet::CardSetPtr)-1 constexpr because
static constexpr ContainerPtr FreeCardSet = nullptr;
// Unfortunately we can't make (G1CardSet::ContainerPtr)-1 constexpr because
// reinterpret_casts are forbidden in constexprs. Use a regular static instead.
static CardSetPtr FullCardSet;
static ContainerPtr FullCardSet;
static const uintptr_t CardSetPtrTypeMask = ((uintptr_t)1 << CardSetPtrHeaderSize) - 1;
static const uintptr_t ContainerPtrTypeMask = ((uintptr_t)1 << ContainerPtrHeaderSize) - 1;
static CardSetPtr strip_card_set_type(CardSetPtr ptr) { return (CardSetPtr)((uintptr_t)ptr & ~CardSetPtrTypeMask); }
static ContainerPtr strip_container_type(ContainerPtr ptr) { return (ContainerPtr)((uintptr_t)ptr & ~ContainerPtrTypeMask); }
static uint card_set_type(CardSetPtr ptr) { return (uintptr_t)ptr & CardSetPtrTypeMask; }
static uint container_type(ContainerPtr ptr) { return (uintptr_t)ptr & ContainerPtrTypeMask; }
template <class T>
static T* card_set_ptr(CardSetPtr ptr);
static T* container_ptr(ContainerPtr ptr);
private:
G1CardSetMemoryManager* _mm;
@ -245,42 +242,42 @@ private:
// be (slightly) more cards in the card set than this value in reality.
size_t _num_occupied;
CardSetPtr make_card_set_ptr(void* value, uintptr_t type);
ContainerPtr make_container_ptr(void* value, uintptr_t type);
CardSetPtr acquire_card_set(CardSetPtr volatile* card_set_addr);
// Returns true if the card set should be released
bool release_card_set(CardSetPtr card_set);
ContainerPtr acquire_container(ContainerPtr volatile* container_addr);
// Returns true if the card set container should be released
bool release_container(ContainerPtr container);
// Release card set and free if needed.
void release_and_maybe_free_card_set(CardSetPtr card_set);
void release_and_maybe_free_container(ContainerPtr container);
// Release card set and free (and it must be freeable).
void release_and_must_free_card_set(CardSetPtr card_set);
void release_and_must_free_container(ContainerPtr container);
// Coarsens the CardSet cur_card_set to the next level; tries to replace the
// previous CardSet with a new one which includes the given card_in_region.
// coarsen_card_set does not transfer cards from cur_card_set
// to the new card_set. Transfer is achieved by transfer_cards.
// Returns true if this was the thread that coarsened the CardSet (and added the card).
bool coarsen_card_set(CardSetPtr volatile* card_set_addr,
CardSetPtr cur_card_set,
uint card_in_region, bool within_howl = false);
// Coarsens the card set container cur_container to the next level; tries to replace the
// previous ContainerPtr with a new one which includes the given card_in_region.
// coarsen_container does not transfer cards from cur_container
// to the new container. Transfer is achieved by transfer_cards.
// Returns true if this was the thread that coarsened the container (and added the card).
bool coarsen_container(ContainerPtr volatile* container_addr,
ContainerPtr cur_container,
uint card_in_region, bool within_howl = false);
CardSetPtr create_coarsened_array_of_cards(uint card_in_region, bool within_howl);
ContainerPtr create_coarsened_array_of_cards(uint card_in_region, bool within_howl);
// Transfer entries from source_card_set to a recently installed coarser storage type
// We only need to transfer anything finer than CardSetBitMap. "Full" contains
// We only need to transfer anything finer than ContainerBitMap. "Full" contains
// all elements anyway.
void transfer_cards(G1CardSetHashTableValue* table_entry, CardSetPtr source_card_set, uint card_region);
void transfer_cards_in_howl(CardSetPtr parent_card_set, CardSetPtr source_card_set, uint card_region);
void transfer_cards(G1CardSetHashTableValue* table_entry, ContainerPtr source_container, uint card_region);
void transfer_cards_in_howl(ContainerPtr parent_container, ContainerPtr source_container, uint card_region);
G1AddCardResult add_to_card_set(CardSetPtr volatile* card_set_addr, CardSetPtr card_set, uint card_region, uint card, bool increment_total = true);
G1AddCardResult add_to_container(ContainerPtr volatile* container_addr, ContainerPtr container, uint card_region, uint card, bool increment_total = true);
G1AddCardResult add_to_inline_ptr(CardSetPtr volatile* card_set_addr, CardSetPtr card_set, uint card_in_region);
G1AddCardResult add_to_array(CardSetPtr card_set, uint card_in_region);
G1AddCardResult add_to_bitmap(CardSetPtr card_set, uint card_in_region);
G1AddCardResult add_to_howl(CardSetPtr parent_card_set, uint card_region, uint card_in_region, bool increment_total = true);
G1AddCardResult add_to_inline_ptr(ContainerPtr volatile* container_addr, ContainerPtr container, uint card_in_region);
G1AddCardResult add_to_array(ContainerPtr container, uint card_in_region);
G1AddCardResult add_to_bitmap(ContainerPtr container, uint card_in_region);
G1AddCardResult add_to_howl(ContainerPtr parent_container, uint card_region, uint card_in_region, bool increment_total = true);
G1CardSetHashTableValue* get_or_add_card_set(uint card_region, bool* should_grow_table);
G1CardSetHashTableValue* get_card_set(uint card_region);
G1CardSetHashTableValue* get_or_add_container(uint card_region, bool* should_grow_table);
G1CardSetHashTableValue* get_container(uint card_region);
// Iterate over cards of a card set container during transfer of the cards from
// one container to another. Executes
@ -289,11 +286,11 @@ private:
//
// on the given class.
template <class CardVisitor>
void iterate_cards_during_transfer(CardSetPtr const card_set, CardVisitor& vl);
void iterate_cards_during_transfer(ContainerPtr const container, CardVisitor& vl);
uint card_set_type_to_mem_object_type(uintptr_t type) const;
uint container_type_to_mem_object_type(uintptr_t type) const;
uint8_t* allocate_mem_object(uintptr_t type);
void free_mem_object(CardSetPtr card_set);
void free_mem_object(ContainerPtr container);
public:
G1CardSetConfiguration* config() const { return _config; }
@ -302,8 +299,8 @@ public:
G1CardSet(G1CardSetConfiguration* config, G1CardSetMemoryManager* mm);
virtual ~G1CardSet();
// Adds the given card to this set, returning an appropriate result. If added,
// updates the total count.
// Adds the given card to this set, returning an appropriate result.
// If incremental_count is true and the card has been added, updates the total count.
G1AddCardResult add_card(uint card_region, uint card_in_region, bool increment_total = true);
bool contains_card(uint card_region, uint card_in_region);
@ -351,14 +348,14 @@ public:
// start_iterate().
//
template <class CardOrRangeVisitor>
void iterate_cards_or_ranges_in_container(CardSetPtr const card_set, CardOrRangeVisitor& cl);
void iterate_cards_or_ranges_in_container(ContainerPtr const container, CardOrRangeVisitor& cl);
class CardSetPtrClosure {
class ContainerPtrClosure {
public:
virtual void do_cardsetptr(uint region_idx, size_t num_occupied, CardSetPtr card_set) = 0;
virtual void do_containerptr(uint region_idx, size_t num_occupied, ContainerPtr container) = 0;
};
void iterate_containers(CardSetPtrClosure* cl, bool safepoint = false);
void iterate_containers(ContainerPtrClosure* cl, bool safepoint = false);
class CardClosure {
public:
@ -370,13 +367,13 @@ public:
class G1CardSetHashTableValue {
public:
using CardSetPtr = G1CardSet::CardSetPtr;
using ContainerPtr = G1CardSet::ContainerPtr;
const uint _region_idx;
uint volatile _num_occupied;
CardSetPtr volatile _card_set;
ContainerPtr volatile _container;
G1CardSetHashTableValue(uint region_idx, CardSetPtr card_set) : _region_idx(region_idx), _num_occupied(0), _card_set(card_set) { }
G1CardSetHashTableValue(uint region_idx, ContainerPtr container) : _region_idx(region_idx), _num_occupied(0), _container(container) { }
};
class G1CardSetHashTableConfig : public StackObj {
@ -391,6 +388,6 @@ public:
static void free_node(void* context, void* memory, Value const& value);
};
typedef ConcurrentHashTable<G1CardSetHashTableConfig, mtGCCardSet> CardSetHash;
using CardSetHash = ConcurrentHashTable<G1CardSetHashTableConfig, mtGCCardSet>;
#endif // SHARE_GC_G1_G1CARDSET_HPP

35
src/hotspot/share/gc/g1/g1CardSet.inline.hpp
View File

@ -28,55 +28,54 @@
#include "gc/g1/g1CardSet.hpp"
#include "gc/g1/g1CardSetContainers.inline.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
#include "runtime/atomic.hpp"
#include "logging/log.hpp"
template <class T>
inline T* G1CardSet::card_set_ptr(CardSetPtr ptr) {
return (T*)strip_card_set_type(ptr);
inline T* G1CardSet::container_ptr(ContainerPtr ptr) {
return (T*)strip_container_type(ptr);
}
inline G1CardSet::CardSetPtr G1CardSet::make_card_set_ptr(void* value, uintptr_t type) {
assert(card_set_type(value) == 0, "Given ptr " PTR_FORMAT " already has type bits set", p2i(value));
return (CardSetPtr)((uintptr_t)value | type);
inline G1CardSet::ContainerPtr G1CardSet::make_container_ptr(void* value, uintptr_t type) {
assert(container_type(value) == 0, "Given ptr " PTR_FORMAT " already has type bits set", p2i(value));
return (ContainerPtr)((uintptr_t)value | type);
}
template <class CardOrRangeVisitor>
inline void G1CardSet::iterate_cards_or_ranges_in_container(CardSetPtr const card_set, CardOrRangeVisitor& cl) {
switch (card_set_type(card_set)) {
case CardSetInlinePtr: {
inline void G1CardSet::iterate_cards_or_ranges_in_container(ContainerPtr const container, CardOrRangeVisitor& cl) {
switch (container_type(container)) {
case ContainerInlinePtr: {
if (cl.start_iterate(G1GCPhaseTimes::MergeRSMergedInline)) {
G1CardSetInlinePtr ptr(card_set);
G1CardSetInlinePtr ptr(container);
ptr.iterate(cl, _config->inline_ptr_bits_per_card());
}
return;
}
case CardSetArrayOfCards : {
case ContainerArrayOfCards: {
if (cl.start_iterate(G1GCPhaseTimes::MergeRSMergedArrayOfCards)) {
card_set_ptr<G1CardSetArray>(card_set)->iterate(cl);
container_ptr<G1CardSetArray>(container)->iterate(cl);
}
return;
}
case CardSetBitMap: {
case ContainerBitMap: {
// There is no first-level bitmap spanning the whole area.
ShouldNotReachHere();
return;
}
case CardSetHowl: {
assert(card_set_type(FullCardSet) == CardSetHowl, "Must be");
if (card_set == FullCardSet) {
case ContainerHowl: {
assert(container_type(FullCardSet) == ContainerHowl, "Must be");
if (container == FullCardSet) {
if (cl.start_iterate(G1GCPhaseTimes::MergeRSMergedFull)) {
cl(0, _config->max_cards_in_region());
}
return;
}
if (cl.start_iterate(G1GCPhaseTimes::MergeRSMergedHowl)) {
card_set_ptr<G1CardSetHowl>(card_set)->iterate(cl, _config);
container_ptr<G1CardSetHowl>(container)->iterate(cl, _config);
}
return;
}
}
log_error(gc)("Unkown card set type %u", card_set_type(card_set));
log_error(gc)("Unkown card set container type %u", container_type(container));
ShouldNotReachHere();
}

58
src/hotspot/share/gc/g1/g1CardSetContainers.hpp
View File

@ -31,7 +31,7 @@
#include "utilities/bitMap.hpp"
#include "utilities/globalDefinitions.hpp"
// A helper class to encode a few card indexes within a CardSetPtr.
// A helper class to encode a few card indexes within a ContainerPtr.
//
// The pointer value (either 32 or 64 bits) is split into two areas:
//
@ -65,16 +65,16 @@
class G1CardSetInlinePtr : public StackObj {
friend class G1CardSetContainersTest;
typedef G1CardSet::CardSetPtr CardSetPtr;
using ContainerPtr = G1CardSet::ContainerPtr;
CardSetPtr volatile * _value_addr;
CardSetPtr _value;
ContainerPtr volatile * _value_addr;
ContainerPtr _value;
static const uint SizeFieldLen = 3;
static const uint SizeFieldPos = 2;
static const uint HeaderSize = G1CardSet::CardSetPtrHeaderSize + SizeFieldLen;
static const uint HeaderSize = G1CardSet::ContainerPtrHeaderSize + SizeFieldLen;
static const uint BitsInValue = sizeof(CardSetPtr) * BitsPerByte;
static const uint BitsInValue = sizeof(ContainerPtr) * BitsPerByte;
static const uintptr_t SizeFieldMask = (((uint)1 << SizeFieldLen) - 1) << SizeFieldPos;
@ -82,9 +82,9 @@ class G1CardSetInlinePtr : public StackObj {
return (idx * bits_per_card + HeaderSize);
}
static CardSetPtr merge(CardSetPtr orig_value, uint card_in_region, uint idx, uint bits_per_card);
static ContainerPtr merge(ContainerPtr orig_value, uint card_in_region, uint idx, uint bits_per_card);
static uint card_at(CardSetPtr value, uint const idx, uint const bits_per_card) {
static uint card_at(ContainerPtr value, uint const idx, uint const bits_per_card) {
uint8_t card_pos = card_pos_for(idx, bits_per_card);
uint result = ((uintptr_t)value >> card_pos) & (((uintptr_t)1 << bits_per_card) - 1);
return result;
@ -93,14 +93,14 @@ class G1CardSetInlinePtr : public StackObj {
uint find(uint const card_idx, uint const bits_per_card, uint start_at, uint num_cards);
public:
G1CardSetInlinePtr() : _value_addr(nullptr), _value((CardSetPtr)G1CardSet::CardSetInlinePtr) { }
G1CardSetInlinePtr() : _value_addr(nullptr), _value((ContainerPtr)G1CardSet::ContainerInlinePtr) { }
G1CardSetInlinePtr(CardSetPtr value) : _value_addr(nullptr), _value(value) {
assert(G1CardSet::card_set_type(_value) == G1CardSet::CardSetInlinePtr, "Value " PTR_FORMAT " is not a valid G1CardSetInPtr.", p2i(_value));
G1CardSetInlinePtr(ContainerPtr value) : _value_addr(nullptr), _value(value) {
assert(G1CardSet::container_type(_value) == G1CardSet::ContainerInlinePtr, "Value " PTR_FORMAT " is not a valid G1CardSetInlinePtr.", p2i(_value));
}
G1CardSetInlinePtr(CardSetPtr volatile* value_addr, CardSetPtr value) : _value_addr(value_addr), _value(value) {
assert(G1CardSet::card_set_type(_value) == G1CardSet::CardSetInlinePtr, "Value " PTR_FORMAT " is not a valid G1CardSetInPtr.", p2i(_value));
G1CardSetInlinePtr(ContainerPtr volatile* value_addr, ContainerPtr value) : _value_addr(value_addr), _value(value) {
assert(G1CardSet::container_type(_value) == G1CardSet::ContainerInlinePtr, "Value " PTR_FORMAT " is not a valid G1CardSetInlinePtr.", p2i(_value));
}
G1AddCardResult add(uint const card_idx, uint const bits_per_card, uint const max_cards_in_inline_ptr);
@ -110,13 +110,13 @@ public:
template <class CardVisitor>
void iterate(CardVisitor& found, uint const bits_per_card);
operator CardSetPtr () { return _value; }
operator ContainerPtr () { return _value; }
static uint max_cards_in_inline_ptr(uint bits_per_card) {
return (BitsInValue - HeaderSize) / bits_per_card;
}
static uint num_cards_in(CardSetPtr value) {
static uint num_cards_in(ContainerPtr value) {
return ((uintptr_t)value & SizeFieldMask) >> SizeFieldPos;
}
};
@ -139,7 +139,7 @@ public:
// which requires that we increment the reference counts by 2 starting at _ref_count = 3.
//
// All but inline pointers are of this kind. For those, card entries are stored
// directly in the CardSetPtr of the ConcurrentHashTable node.
// directly in the ContainerPtr of the ConcurrentHashTable node.
class G1CardSetContainer {
uintptr_t _ref_count;
protected:
@ -163,7 +163,7 @@ class G1CardSetArray : public G1CardSetContainer {
public:
typedef uint16_t EntryDataType;
typedef uint EntryCountType;
using CardSetPtr = G1CardSet::CardSetPtr;
using ContainerPtr = G1CardSet::ContainerPtr;
private:
EntryCountType _size;
EntryCountType volatile _num_entries;
@ -217,7 +217,7 @@ class G1CardSetBitMap : public G1CardSetContainer {
size_t _num_bits_set;
BitMap::bm_word_t _bits[1];
using CardSetPtr = G1CardSet::CardSetPtr;
using ContainerPtr = G1CardSet::ContainerPtr;
template<typename Derived>
static size_t header_size_in_bytes_internal() {
@ -252,10 +252,10 @@ public:
class G1CardSetHowl : public G1CardSetContainer {
public:
typedef uint EntryCountType;
using CardSetPtr = G1CardSet::CardSetPtr;
using ContainerPtr = G1CardSet::ContainerPtr;
EntryCountType volatile _num_entries;
private:
CardSetPtr _buckets[2];
ContainerPtr _buckets[2];
// Do not add class member variables beyond this point
template<typename Derived>
@ -263,32 +263,32 @@ private:
return offset_of(Derived, _buckets);
}
// Iterates over the given CardSetPtr with at index in this Howl card set,
// Iterates over the given ContainerPtr with at index in this Howl card set,
// applying a CardOrRangeVisitor on it.
template <class CardOrRangeVisitor>
void iterate_cardset(CardSetPtr const card_set, uint index, CardOrRangeVisitor& found, G1CardSetConfiguration* config);
void iterate_cardset(ContainerPtr const container, uint index, CardOrRangeVisitor& found, G1CardSetConfiguration* config);
public:
G1CardSetHowl(EntryCountType card_in_region, G1CardSetConfiguration* config);
CardSetPtr* get_card_set_addr(EntryCountType index) {
ContainerPtr* get_container_addr(EntryCountType index) {
return &_buckets[index];
}
bool contains(uint card_idx, G1CardSetConfiguration* config);
// Iterates over all CardSetPtrs in this Howl card set, applying a CardOrRangeVisitor
// Iterates over all ContainerPtrs in this Howl card set, applying a CardOrRangeVisitor
// on it.
template <class CardOrRangeVisitor>
void iterate(CardOrRangeVisitor& found, G1CardSetConfiguration* config);
// Iterates over all CardSetPtrs in this Howl card set. Calls
// Iterates over all ContainerPtrs in this Howl card set. Calls
//
// void operator ()(CardSetPtr* card_set_addr);
// void operator ()(ContainerPtr* card_set_addr);
//
// on all of them.
template <class CardSetPtrVisitor>
void iterate(CardSetPtrVisitor& found, uint num_card_sets);
template <class ContainerPtrVisitor>
void iterate(ContainerPtrVisitor& found, uint num_card_sets);
static EntryCountType num_buckets(size_t size_in_bits, size_t num_cards_in_array, size_t max_buckets);
@ -300,7 +300,7 @@ public:
static size_t header_size_in_bytes() { return header_size_in_bytes_internal<G1CardSetHowl>(); }
static size_t size_in_bytes(size_t num_arrays) {
return header_size_in_bytes() + sizeof(CardSetPtr) * num_arrays;
return header_size_in_bytes() + sizeof(ContainerPtr) * num_arrays;
}
};

56
src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp
View File

@ -31,7 +31,7 @@
#include "utilities/globalDefinitions.hpp"
#include "utilities/spinYield.hpp"
inline G1CardSetInlinePtr::CardSetPtr G1CardSetInlinePtr::merge(CardSetPtr orig_value, uint card_in_region, uint idx, uint bits_per_card) {
inline G1CardSetInlinePtr::ContainerPtr G1CardSetInlinePtr::merge(ContainerPtr orig_value, uint card_in_region, uint idx, uint bits_per_card) {
assert((idx & (SizeFieldMask >> SizeFieldPos)) == idx, "Index %u too large to fit into size field", idx);
assert(card_in_region < ((uint)1 << bits_per_card), "Card %u too large to fit into card value field", card_in_region);
@ -44,7 +44,7 @@ inline G1CardSetInlinePtr::CardSetPtr G1CardSetInlinePtr::merge(CardSetPtr orig_
uintptr_t value = ((uintptr_t)(idx + 1) << SizeFieldPos) | ((uintptr_t)card_in_region << card_pos);
uintptr_t res = (((uintptr_t)orig_value & ~SizeFieldMask) | value);
return (CardSetPtr)res;
return (ContainerPtr)res;
}
inline G1AddCardResult G1CardSetInlinePtr::add(uint card_idx, uint bits_per_card, uint max_cards_in_inline_ptr) {
@ -64,8 +64,8 @@ inline G1AddCardResult G1CardSetInlinePtr::add(uint card_idx, uint bits_per_card
if (num_cards >= max_cards_in_inline_ptr) {
return Overflow;
}
CardSetPtr new_value = merge(_value, card_idx, num_cards, bits_per_card);
CardSetPtr old_value = Atomic::cmpxchg(_value_addr, _value, new_value, memory_order_relaxed);
ContainerPtr new_value = merge(_value, card_idx, num_cards, bits_per_card);
ContainerPtr old_value = Atomic::cmpxchg(_value_addr, _value, new_value, memory_order_relaxed);
if (_value == old_value) {
return Added;
}
@ -73,7 +73,7 @@ inline G1AddCardResult G1CardSetInlinePtr::add(uint card_idx, uint bits_per_card
_value = old_value;
// The value of the pointer may have changed to something different than
// an inline card set. Exit then instead of overwriting.
if (G1CardSet::card_set_type(_value) != G1CardSet::CardSetInlinePtr) {
if (G1CardSet::container_type(_value) != G1CardSet::ContainerInlinePtr) {
return Overflow;
}
}
@ -268,23 +268,23 @@ inline G1CardSetHowl::G1CardSetHowl(EntryCountType card_in_region, G1CardSetConf
inline bool G1CardSetHowl::contains(uint card_idx, G1CardSetConfiguration* config) {
EntryCountType bucket = config->howl_bucket_index(card_idx);
CardSetPtr* array_entry = get_card_set_addr(bucket);
CardSetPtr card_set = Atomic::load_acquire(array_entry);
ContainerPtr* array_entry = get_container_addr(bucket);
ContainerPtr container = Atomic::load_acquire(array_entry);
switch (G1CardSet::card_set_type(card_set)) {
case G1CardSet::CardSetArrayOfCards : {
return G1CardSet::card_set_ptr<G1CardSetArray>(card_set)->contains(card_idx);
switch (G1CardSet::container_type(container)) {
case G1CardSet::ContainerArrayOfCards: {
return G1CardSet::container_ptr<G1CardSetArray>(container)->contains(card_idx);
}
case G1CardSet::CardSetBitMap: {
case G1CardSet::ContainerBitMap: {
uint card_offset = config->howl_bitmap_offset(card_idx);
return G1CardSet::card_set_ptr<G1CardSetBitMap>(card_set)->contains(card_offset, config->max_cards_in_howl_bitmap());
return G1CardSet::container_ptr<G1CardSetBitMap>(container)->contains(card_offset, config->max_cards_in_howl_bitmap());
}
case G1CardSet::CardSetInlinePtr: {
G1CardSetInlinePtr ptr(card_set);
case G1CardSet::ContainerInlinePtr: {
G1CardSetInlinePtr ptr(container);
return ptr.contains(card_idx, config->inline_ptr_bits_per_card());
}
case G1CardSet::CardSetHowl: {// Fullcard set entry
assert(card_set == G1CardSet::FullCardSet, "Must be");
case G1CardSet::ContainerHowl: {// Fullcard set entry
assert(container == G1CardSet::FullCardSet, "Must be");
return true;
}
}
@ -298,38 +298,38 @@ inline void G1CardSetHowl::iterate(CardOrRangeVisitor& found, G1CardSetConfigura
}
}
template <class CardSetPtrVisitor>
inline void G1CardSetHowl::iterate(CardSetPtrVisitor& found, uint num_card_sets) {
template <class ContainerPtrVisitor>
inline void G1CardSetHowl::iterate(ContainerPtrVisitor& found, uint num_card_sets) {
for (uint i = 0; i < num_card_sets; ++i) {
found(&_buckets[i]);
}
}
template <class CardOrRangeVisitor>
inline void G1CardSetHowl::iterate_cardset(CardSetPtr const card_set, uint index, CardOrRangeVisitor& found, G1CardSetConfiguration* config) {
switch (G1CardSet::card_set_type(card_set)) {
case G1CardSet::CardSetInlinePtr: {
inline void G1CardSetHowl::iterate_cardset(ContainerPtr const container, uint index, CardOrRangeVisitor& found, G1CardSetConfiguration* config) {
switch (G1CardSet::container_type(container)) {
case G1CardSet::ContainerInlinePtr: {
if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlInline)) {
G1CardSetInlinePtr ptr(card_set);
G1CardSetInlinePtr ptr(container);
ptr.iterate(found, config->inline_ptr_bits_per_card());
}
return;
}
case G1CardSet::CardSetArrayOfCards : {
case G1CardSet::ContainerArrayOfCards: {
if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlArrayOfCards)) {
G1CardSet::card_set_ptr<G1CardSetArray>(card_set)->iterate(found);
G1CardSet::container_ptr<G1CardSetArray>(container)->iterate(found);
}
return;
}
case G1CardSet::CardSetBitMap: {
case G1CardSet::ContainerBitMap: {
if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlBitmap)) {
uint offset = index << config->log2_max_cards_in_howl_bitmap();
G1CardSet::card_set_ptr<G1CardSetBitMap>(card_set)->iterate(found, config->max_cards_in_howl_bitmap(), offset);
G1CardSet::container_ptr<G1CardSetBitMap>(container)->iterate(found, config->max_cards_in_howl_bitmap(), offset);
}
return;
}
case G1CardSet::CardSetHowl: { // actually FullCardSet
assert(card_set == G1CardSet::FullCardSet, "Must be");
case G1CardSet::ContainerHowl: { // actually FullCardSet
assert(container == G1CardSet::FullCardSet, "Must be");
if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlFull)) {
uint offset = index << config->log2_max_cards_in_howl_bitmap();
found(offset, config->max_cards_in_howl_bitmap());

6
src/hotspot/share/gc/g1/heapRegionRemSet.inline.hpp
View File

@ -78,7 +78,7 @@ public:
};
template <typename Closure, template <typename> class CardOrRanges>
class G1HeapRegionRemSetMergeCardClosure : public G1CardSet::CardSetPtrClosure {
class G1HeapRegionRemSetMergeCardClosure : public G1CardSet::ContainerPtrClosure {
G1CardSet* _card_set;
Closure& _cl;
uint _log_card_regions_per_region;
@ -98,11 +98,11 @@ public:
_log_card_region_size(log_card_region_size) {
}
void do_cardsetptr(uint card_region_idx, size_t num_occupied, G1CardSet::CardSetPtr card_set) override {
void do_containerptr(uint card_region_idx, size_t num_occupied, G1CardSet::ContainerPtr container) override {
CardOrRanges<Closure> cl(_cl,
card_region_idx >> _log_card_regions_per_region,
(card_region_idx & _card_regions_per_region_mask) << _log_card_region_size);
_card_set->iterate_cards_or_ranges_in_container(card_set, cl);
_card_set->iterate_cards_or_ranges_in_container(container, cl);
}
};

4
test/hotspot/gtest/gc/g1/test_g1CardSet.cpp
View File

@ -165,13 +165,13 @@ void G1CardSetTest::translate_cards(uint cards_per_region, uint region_idx, uint
}
}
class G1CountCardsOccupied : public G1CardSet::CardSetPtrClosure {
class G1CountCardsOccupied : public G1CardSet::ContainerPtrClosure {
size_t _num_occupied;
public:
G1CountCardsOccupied() : _num_occupied(0) { }
void do_cardsetptr(uint region_idx, size_t num_occupied, G1CardSet::CardSetPtr card_set) override {
void do_containerptr(uint region_idx, size_t num_occupied, G1CardSet::ContainerPtr container) override {
_num_occupied += num_occupied;
}

2
test/hotspot/gtest/gc/g1/test_g1CardSetContainers.cpp
View File

@ -83,7 +83,7 @@ void G1CardSetContainersTest::cardset_inlineptr_test(uint bits_per_card) {
G1AddCardResult res;
G1CardSet::CardSetPtr value = G1CardSetInlinePtr();
G1CardSet::ContainerPtr value = G1CardSetInlinePtr();
for (uint i = 0; i < CardsPerSet; i++) {
{