8017163: G1: Refactor remembered sets

8048504: G1: Investigate replacing the coarse and fine grained data structures in the remembered sets
6949259: G1: Merge sparse and fine remembered set hash tables

Co-authored-by: Ivan Walulya <iwalulya@openjdk.org>
Co-authored-by: Thomas Schatzl <tschatzl@openjdk.org>
Reviewed-by: sjohanss, iwalulya

src/hotspot/share/gc/g1/g1Arguments.cpp

@@ -25,6 +25,8 @@
 
 #include "precompiled.hpp"
 #include "gc/g1/g1Arguments.hpp"
+#include "gc/g1/g1CardSet.hpp"
+#include "gc/g1/g1CardSetContainers.inline.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1HeapVerifier.hpp"
 #include "gc/g1/heapRegion.hpp"
@@ -50,16 +52,17 @@ void G1Arguments::initialize_alignments() {
   // around this we use the unaligned values for the heap.
   HeapRegion::setup_heap_region_size(MaxHeapSize);
 
-  // The remembered set needs the heap regions set up.
-  HeapRegionRemSet::setup_remset_size();
+  SpaceAlignment = HeapRegion::GrainBytes;
+  HeapAlignment = calculate_heap_alignment(SpaceAlignment);
+
+  // We need to initialize card set configuration as soon as heap region size is
+  // known as it depends on it and is used really early.
+  initialize_card_set_configuration();
   // Needs remembered set initialization as the ergonomics are based
   // on it.
   if (FLAG_IS_DEFAULT(G1EagerReclaimRemSetThreshold)) {
-    FLAG_SET_ERGO(G1EagerReclaimRemSetThreshold, G1RSetSparseRegionEntries);
+    FLAG_SET_ERGO(G1EagerReclaimRemSetThreshold, G1RemSetArrayOfCardsEntries);
   }
-
-  SpaceAlignment = HeapRegion::GrainBytes;
-  HeapAlignment = calculate_heap_alignment(SpaceAlignment);
 }
 
 size_t G1Arguments::conservative_max_heap_alignment() {
@@ -119,6 +122,40 @@ void G1Arguments::initialize_mark_stack_size() {
   log_trace(gc)("MarkStackSize: %uk  MarkStackSizeMax: %uk", (uint)(MarkStackSize / K), (uint)(MarkStackSizeMax / K));
 }
 
+
+void G1Arguments::initialize_card_set_configuration() {
+  assert(HeapRegion::LogOfHRGrainBytes != 0, "not initialized");
+  // Array of Cards card set container globals.
+  const int LOG_M = 20;
+  uint region_size_log_mb = (uint)MAX2(HeapRegion::LogOfHRGrainBytes - LOG_M, 0);
+
+  if (FLAG_IS_DEFAULT(G1RemSetArrayOfCardsEntries)) {
+    uint num_cards_in_inline_ptr = G1CardSetConfiguration::num_cards_in_inline_ptr(HeapRegion::LogOfHRGrainBytes - CardTable::card_shift);
+    FLAG_SET_ERGO(G1RemSetArrayOfCardsEntries, MAX2(num_cards_in_inline_ptr * 2,
+                                                    G1RemSetArrayOfCardsEntriesBase * (1u << (region_size_log_mb + 1))));
+  }
+
+  // Round to next 8 byte boundary for array to maximize space usage.
+  size_t const cur_size = G1CardSetArray::size_in_bytes(G1RemSetArrayOfCardsEntries);
+  FLAG_SET_ERGO(G1RemSetArrayOfCardsEntries,
+                G1RemSetArrayOfCardsEntries + (uint)(align_up(cur_size, G1CardSetAllocOptions::BufferAlignment) - cur_size) / sizeof(G1CardSetArray::EntryDataType));
+
+  // Howl card set container globals.
+  if (FLAG_IS_DEFAULT(G1RemSetHowlNumBuckets)) {
+    FLAG_SET_ERGO(G1RemSetHowlNumBuckets, G1CardSetHowl::num_buckets(HeapRegion::CardsPerRegion,
+                                                                     G1RemSetArrayOfCardsEntries,
+                                                                     G1RemSetHowlMaxNumBuckets));
+  }
+
+  if (FLAG_IS_DEFAULT(G1RemSetHowlMaxNumBuckets)) {
+    FLAG_SET_ERGO(G1RemSetHowlMaxNumBuckets, MAX2(G1RemSetHowlMaxNumBuckets, G1RemSetHowlNumBuckets));
+  } else if (G1RemSetHowlMaxNumBuckets < G1RemSetHowlNumBuckets) {
+    FormatBuffer<> buf("Maximum Howl card set container bucket size %u smaller than requested bucket size %u",
+                       G1RemSetHowlMaxNumBuckets, G1RemSetHowlNumBuckets);
+    vm_exit_during_initialization(buf);
+  }
+}
+
 void G1Arguments::initialize() {
   GCArguments::initialize();
   assert(UseG1GC, "Error");
@@ -196,6 +233,14 @@ void G1Arguments::initialize() {
 
   initialize_mark_stack_size();
   initialize_verification_types();
+
+  // Verify that the maximum parallelism isn't too high to eventually overflow
+  // the refcount in G1CardSetContainer.
+  uint max_parallel_refinement_threads = G1ConcRefinementThreads + G1DirtyCardQueueSet::num_par_ids();
+  uint const divisor = 3;  // Safe divisor; we increment by 2 for each claim, but there is a small initial value.
+  if (max_parallel_refinement_threads > UINTPTR_MAX / divisor) {
+    vm_exit_during_initialization("Too large parallelism for remembered sets.");
+  }
 }
 
 void G1Arguments::initialize_heap_flags_and_sizes() {

src/hotspot/share/gc/g1/g1Arguments.hpp

@@ -34,6 +34,7 @@ class G1Arguments : public GCArguments {
   friend class G1HeapVerifierTest;
 
   static void initialize_mark_stack_size();
+  static void initialize_card_set_configuration();
   static void initialize_verification_types();
   static void parse_verification_type(const char* type);
 

src/hotspot/share/gc/g1/g1CardSet.cpp

@@ -0,0 +1,887 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/g1/g1CardSet.inline.hpp"
+#include "gc/g1/g1CardSetContainers.inline.hpp"
+#include "gc/g1/g1CardSetMemory.inline.hpp"
+#include "gc/g1/g1FromCardCache.hpp"
+#include "gc/g1/heapRegion.inline.hpp"
+#include "memory/allocation.inline.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/globals_extension.hpp"
+#include "runtime/mutex.hpp"
+#include "utilities/bitMap.inline.hpp"
+#include "utilities/concurrentHashTable.inline.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/lockFreeStack.hpp"
+#include "utilities/spinYield.hpp"
+
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/shared/gcTraceTime.inline.hpp"
+#include "runtime/java.hpp"
+
+G1CardSet::CardSetPtr G1CardSet::FullCardSet = (G1CardSet::CardSetPtr)-1;
+
+G1CardSetConfiguration::G1CardSetConfiguration() :
+  _inline_ptr_bits_per_card(HeapRegion::LogOfHRGrainBytes - CardTable::card_shift) {
+
+  // Array of Cards card set container size calculation
+  _num_cards_in_array = G1RemSetArrayOfCardsEntries;
+
+  // Full card set container size calculation
+  _max_cards_in_card_set = (uint)HeapRegion::CardsPerRegion;
+  assert(is_power_of_2(_max_cards_in_card_set),
+        "max_cards_in_card_set must be a power of 2: %u", _max_cards_in_card_set);
+  _cards_in_howl_threshold = _max_cards_in_card_set * (double)G1RemSetCoarsenHowlToFullPercent / 100;
+
+  // Howl card set container size calculation.
+  _num_buckets_in_howl = G1RemSetHowlNumBuckets;
+
+  // Howl Bitmap card set container size calculation.
+  _num_cards_in_howl_bitmap = G1CardSetHowl::bitmap_size(_max_cards_in_card_set, _num_buckets_in_howl);
+  _log2_num_cards_in_howl_bitmap = log2i_exact(_num_cards_in_howl_bitmap);
+  _cards_in_howl_bitmap_threshold = _num_cards_in_howl_bitmap * (double)G1RemSetCoarsenHowlBitmapToHowlFullPercent / 100;
+  _bitmap_hash_mask = ~(~(0) << _log2_num_cards_in_howl_bitmap);
+
+  log_configuration();
+}
+
+G1CardSetConfiguration::G1CardSetConfiguration(uint inline_ptr_bits_per_card,
+                                               uint num_cards_in_array,
+                                               double cards_in_bitmap_threshold,
+                                               uint max_buckets_in_howl,
+                                               double cards_in_howl_threshold,
+                                               uint max_cards_in_cardset) :
+  _inline_ptr_bits_per_card(inline_ptr_bits_per_card),
+  _num_cards_in_array(num_cards_in_array),
+  _max_cards_in_card_set(max_cards_in_cardset),
+  _cards_in_howl_threshold(max_cards_in_cardset * cards_in_howl_threshold) {
+
+  assert(is_power_of_2(_max_cards_in_card_set),
+        "max_cards_in_card_set must be a power of 2: %u", _max_cards_in_card_set);
+
+  _num_buckets_in_howl = G1CardSetHowl::num_buckets(_max_cards_in_card_set, _num_cards_in_array, max_buckets_in_howl);
+
+  _num_cards_in_howl_bitmap = G1CardSetHowl::bitmap_size(_max_cards_in_card_set, _num_buckets_in_howl);
+  _cards_in_howl_bitmap_threshold = _num_cards_in_howl_bitmap * cards_in_bitmap_threshold;
+  _log2_num_cards_in_howl_bitmap = log2i_exact(_num_cards_in_howl_bitmap);
+  _bitmap_hash_mask = ~(~(0) << _log2_num_cards_in_howl_bitmap);
+
+  log_configuration();
+}
+
+void G1CardSetConfiguration::log_configuration() {
+  log_debug_p(gc, remset)("Card Set container configuration: "
+                          "InlinePtr #elems %u size %zu "
+                          "Array Of Cards #elems %u size %zu "
+                          "Howl #buckets %u coarsen threshold %u "
+                          "Howl Bitmap #elems %u size %zu coarsen threshold %u",
+                          num_cards_in_inline_ptr(), sizeof(void*),
+                          num_cards_in_array(), G1CardSetArray::size_in_bytes(num_cards_in_array()),
+                          num_buckets_in_howl(), cards_in_howl_threshold(),
+                          num_cards_in_howl_bitmap(), G1CardSetBitMap::size_in_bytes(num_cards_in_howl_bitmap()), cards_in_howl_bitmap_threshold());
+}
+
+uint G1CardSetConfiguration::num_cards_in_inline_ptr() const {
+  return num_cards_in_inline_ptr(_inline_ptr_bits_per_card);
+}
+
+uint G1CardSetConfiguration::num_cards_in_inline_ptr(uint bits_per_card) {
+  return G1CardSetInlinePtr::max_cards_in_inline_ptr(bits_per_card);
+}
+
+G1CardSetAllocOptions* G1CardSetConfiguration::mem_object_alloc_options() {
+  G1CardSetAllocOptions* result = NEW_C_HEAP_ARRAY(G1CardSetAllocOptions, num_mem_object_types(), mtGC);
+
+  result[0] = { (uint)CardSetHash::get_node_size() };
+  result[1] = { (uint)G1CardSetArray::size_in_bytes(num_cards_in_array()), 2, 256 };
+  result[2] = { (uint)G1CardSetBitMap::size_in_bytes(num_cards_in_howl_bitmap()), 2, 256 };
+  result[3] = { (uint)G1CardSetHowl::size_in_bytes(num_buckets_in_howl()), 2, 256 };
+
+  return result;
+}
+
+const char* G1CardSetConfiguration::mem_object_type_name_str(uint index) {
+  const char* names[] = { "Node", "Array", "Bitmap", "Howl" };
+  return names[index];
+}
+
+void G1CardSetCoarsenStats::reset() {
+  STATIC_ASSERT(ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision));
+  for (uint i = 0; i < ARRAY_SIZE(_coarsen_from); i++) {
+    _coarsen_from[i] = 0;
+    _coarsen_collision[i] = 0;
+  }
+}
+
+void G1CardSetCoarsenStats::subtract_from(G1CardSetCoarsenStats& other) {
+  STATIC_ASSERT(ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision));
+  for (uint i = 0; i < ARRAY_SIZE(_coarsen_from); i++) {
+    _coarsen_from[i] = other._coarsen_from[i] - _coarsen_from[i];
+    _coarsen_collision[i] = other._coarsen_collision[i] - _coarsen_collision[i];
+  }
+}
+
+void G1CardSetCoarsenStats::record_coarsening(uint tag, bool collision) {
+  assert(tag < ARRAY_SIZE(_coarsen_from), "tag %u out of bounds", tag);
+  Atomic::inc(&_coarsen_from[tag], memory_order_relaxed);
+  if (collision) {
+    Atomic::inc(&_coarsen_collision[tag], memory_order_relaxed);
+  }
+}
+
+void G1CardSetCoarsenStats::print_on(outputStream* out) {
+  out->print_cr("Inline->AoC %zu (%zu) "
+                "AoC->Howl %zu (%zu) "
+                "Howl->Full %zu (%zu) "
+                "Inline->AoC %zu (%zu) "
+                "AoC->BitMap %zu (%zu) "
+                "BitMap->Full %zu (%zu) ",
+                _coarsen_from[0], _coarsen_collision[0],
+                _coarsen_from[1], _coarsen_collision[1],
+                // There is no BitMap at the first level so we can't .
+                _coarsen_from[3], _coarsen_collision[3],
+                _coarsen_from[4], _coarsen_collision[4],
+                _coarsen_from[5], _coarsen_collision[5],
+                _coarsen_from[6], _coarsen_collision[6]
+               );
+}
+
+class G1CardSetHashTable : public CHeapObj<mtGCCardSet> {
+  using CardSetPtr = G1CardSet::CardSetPtr;
+
+  // Did we insert at least one element in the table?
+  bool volatile _inserted_elem;
+
+  G1CardSetMemoryManager* _mm;
+  CardSetHash _table;
+
+  class G1CardSetHashTableLookUp : public StackObj {
+    uint _region_idx;
+  public:
+    explicit G1CardSetHashTableLookUp(uint region_idx) : _region_idx(region_idx) { }
+
+    uintx get_hash() const { return _region_idx; }
+
+    bool equals(G1CardSetHashTableValue* value, bool* is_dead) {
+      *is_dead = false;
+      return value->_region_idx == _region_idx;
+    }
+  };
+
+  class G1CardSetHashTableFound : public StackObj {
+    G1CardSetHashTableValue* _value;
+  public:
+    void operator()(G1CardSetHashTableValue* value) {
+      _value = value;
+    }
+
+    G1CardSetHashTableValue* value() const { return _value; }
+  };
+
+  class G1CardSetHashTableScan : public StackObj {
+    G1CardSet::G1CardSetPtrIterator* _scan_f;
+  public:
+    explicit G1CardSetHashTableScan(G1CardSet::G1CardSetPtrIterator* f) : _scan_f(f) { }
+
+    bool operator()(G1CardSetHashTableValue* value) {
+      _scan_f->do_cardsetptr(value->_region_idx, value->_num_occupied, value->_card_set);
+      return true;
+    }
+  };
+
+
+public:
+  static const size_t InitialLogTableSize = 2;
+
+  G1CardSetHashTable(G1CardSetMemoryManager* mm,
+                     size_t initial_log_table_size = InitialLogTableSize) :
+    _inserted_elem(false),
+    _mm(mm),
+    _table(mm, initial_log_table_size) {
+  }
+
+  ~G1CardSetHashTable() {
+    reset();
+  }
+
+  G1CardSetHashTableValue* get_or_add(uint region_idx, bool* should_grow) {
+    G1CardSetHashTableLookUp lookup(region_idx);
+    G1CardSetHashTableFound found;
+
+    if (_table.get(Thread::current(), lookup, found)) {
+      return found.value();
+    }
+
+    G1CardSetHashTableValue value(region_idx, G1CardSetInlinePtr());
+    bool inserted = _table.insert_get(Thread::current(), lookup, value, found, should_grow);
+
+    if (!_inserted_elem && inserted) {
+      // It does not matter to us who is setting the flag so a regular atomic store
+      // is sufficient.
+      Atomic::store(&_inserted_elem, true);
+    }
+
+    return found.value();
+  }
+
+  CardSetPtr get(uint region_idx) {
+    G1CardSetHashTableLookUp lookup(region_idx);
+    G1CardSetHashTableFound found;
+
+    if (_table.get(Thread::current(), lookup, found)) {
+      return found.value()->_card_set;
+    }
+    return nullptr;
+  }
+
+  void iterate_safepoint(G1CardSet::G1CardSetPtrIterator* cl2) {
+    G1CardSetHashTableScan cl(cl2);
+    _table.do_safepoint_scan(cl);
+  }
+
+  void iterate(G1CardSet::G1CardSetPtrIterator* cl2) {
+    G1CardSetHashTableScan cl(cl2);
+    _table.do_scan(Thread::current(), cl);
+  }
+
+  void reset() {
+    if (Atomic::load(&_inserted_elem)) {
+       _table.unsafe_reset(InitialLogTableSize);
+      Atomic::store(&_inserted_elem, false);
+    }
+  }
+
+  void print(outputStream* os) {
+    os->print("TBL " PTR_FORMAT " size %zu mem %zu ", p2i(&_table), _table.get_size_log2(Thread::current()), _table.get_mem_size(Thread::current()));
+  }
+
+  void grow() {
+    size_t new_limit = _table.get_size_log2(Thread::current()) + 1;
+    _table.grow(Thread::current(), new_limit);
+  }
+
+  size_t mem_size() {
+    return sizeof(*this) +
+      _table.get_mem_size(Thread::current()) - sizeof(_table);
+  }
+
+  size_t log_table_size() { return _table.get_size_log2(Thread::current()); }
+};
+
+void* G1CardSetHashTableConfig::allocate_node(void* context, size_t size, Value const& value) {
+  G1CardSetMemoryManager* mm = (G1CardSetMemoryManager*)context;
+  return mm->allocate_node();
+}
+
+void G1CardSetHashTableConfig::free_node(void* context, void* memory, Value const& value) {
+  G1CardSetMemoryManager* mm = (G1CardSetMemoryManager*)context;
+  mm->free_node(memory);
+}
+
+G1CardSetCoarsenStats G1CardSet::_coarsen_stats;
+G1CardSetCoarsenStats G1CardSet::_last_coarsen_stats;
+
+G1CardSet::G1CardSet(G1CardSetConfiguration* config, G1CardSetMemoryManager* mm) :
+  _mm(mm),
+  _config(config),
+  _table(new G1CardSetHashTable(mm)),
+  _num_occupied(0) {
+}
+
+G1CardSet::~G1CardSet() {
+  delete _table;
+  _mm->flush();
+}
+
+uint G1CardSet::card_set_type_to_mem_object_type(uintptr_t type) const {
+  assert(type == G1CardSet::CardSetArrayOfCards ||
+         type == G1CardSet::CardSetBitMap ||
+         type == G1CardSet::CardSetHowl, "should not allocate card set type %zu", type);
+
+  return (uint)type;
+}
+
+uint8_t* G1CardSet::allocate_mem_object(uintptr_t type) {
+  return _mm->allocate(card_set_type_to_mem_object_type(type));
+}
+
+void G1CardSet::free_mem_object(CardSetPtr card_set) {
+  assert(card_set != G1CardSet::FreeCardSet, "should not free Free card set");
+  assert(card_set != G1CardSet::FullCardSet, "should not free Full card set");
+
+  uintptr_t type = card_set_type(card_set);
+  void* value = strip_card_set_type(card_set);
+
+  assert(type == G1CardSet::CardSetArrayOfCards ||
+         type == G1CardSet::CardSetBitMap ||
+         type == G1CardSet::CardSetHowl, "should not free card set type %zu", type);
+
+#ifdef ASSERT
+  if (type == G1CardSet::CardSetArrayOfCards ||
+      type == G1CardSet::CardSetBitMap ||
+      type == G1CardSet::CardSetHowl) {
+    G1CardSetContainer* card_set = (G1CardSetContainer*)value;
+    assert((card_set->refcount() == 1), "must be");
+  }
+#endif
+
+  _mm->free(card_set_type_to_mem_object_type(type), value);
+}
+
+G1CardSet::CardSetPtr G1CardSet::acquire_card_set(CardSetPtr volatile* card_set_addr) {
+  // Update reference counts under RCU critical section to avoid a
+  // use-after-cleapup bug where we increment a reference count for
+  // an object whose memory has already been cleaned up and reused.
+  GlobalCounter::CriticalSection cs(Thread::current());
+  while (true) {
+    // Get cardsetptr and increment refcount atomically wrt to memory reuse.
+    CardSetPtr card_set = Atomic::load_acquire(card_set_addr);
+    uint cs_type = card_set_type(card_set);
+    if (card_set == FullCardSet || cs_type == CardSetInlinePtr) {
+      return card_set;
+    }
+
+    G1CardSetContainer* card_set_on_heap = (G1CardSetContainer*)strip_card_set_type(card_set);
+
+    if (card_set_on_heap->try_increment_refcount()) {
+      assert(card_set_on_heap->refcount() >= 3, "Smallest value is 3");
+      return card_set;
+    }
+  }
+}
+
+bool G1CardSet::release_card_set(CardSetPtr card_set) {
+  uint cs_type = card_set_type(card_set);
+  if (card_set == FullCardSet || cs_type == CardSetInlinePtr) {
+    return false;
+  }
+
+  G1CardSetContainer* card_set_on_heap = (G1CardSetContainer*)strip_card_set_type(card_set);
+  return card_set_on_heap->decrement_refcount() == 1;
+}
+
+void G1CardSet::release_and_maybe_free_card_set(CardSetPtr card_set) {
+  if (release_card_set(card_set)) {
+    free_mem_object(card_set);
+  }
+}
+
+void G1CardSet::release_and_must_free_card_set(CardSetPtr card_set) {
+  bool should_free = release_card_set(card_set);
+  assert(should_free, "should have been the only one having a reference");
+  free_mem_object(card_set);
+}
+
+class G1ReleaseCardsets : public StackObj {
+  G1CardSet* _card_set;
+  using CardSetPtr = G1CardSet::CardSetPtr;
+
+  void coarsen_to_full(CardSetPtr* card_set_addr) {
+    while (true) {
+      CardSetPtr cur_card_set = Atomic::load_acquire(card_set_addr);
+      uint cs_type = G1CardSet::card_set_type(cur_card_set);
+      if (cur_card_set == G1CardSet::FullCardSet) {
+        return;
+      }
+
+      CardSetPtr old_value = Atomic::cmpxchg(card_set_addr, cur_card_set, G1CardSet::FullCardSet);
+
+      if (old_value == cur_card_set) {
+        _card_set->release_and_maybe_free_card_set(cur_card_set);
+        return;
+      }
+    }
+  }
+
+public:
+  explicit G1ReleaseCardsets(G1CardSet* card_set) : _card_set(card_set) { }
+
+  void operator ()(CardSetPtr* card_set_addr) {
+    coarsen_to_full(card_set_addr);
+  }
+};
+
+G1AddCardResult G1CardSet::add_to_array(CardSetPtr card_set, uint card_in_region) {
+  G1CardSetArray* array = card_set_ptr<G1CardSetArray>(card_set);
+  return array->add(card_in_region);
+}
+
+G1AddCardResult G1CardSet::add_to_howl(CardSetPtr parent_card_set,
+                                                uint card_region,
+                                                uint card_in_region,
+                                                bool increment_total) {
+  G1CardSetHowl* howl = card_set_ptr<G1CardSetHowl>(parent_card_set);
+
+  G1AddCardResult add_result;
+  CardSetPtr to_transfer = nullptr;
+  CardSetPtr card_set;
+
+  uint bucket = _config->howl_bucket_index(card_in_region);
+  volatile CardSetPtr* bucket_entry = howl->get_card_set_addr(bucket);
+
+  while (true) {
+    if (Atomic::load(&howl->_num_entries) >= _config->cards_in_howl_threshold()) {
+      return Overflow;
+    }
+
+    card_set = acquire_card_set(bucket_entry);
+    add_result = add_to_card_set(bucket_entry, card_set, card_region, card_in_region);
+
+    if (add_result != Overflow) {
+      break;
+    }
+    // Card set has overflown. Coarsen or retry.
+    bool coarsened = coarsen_card_set(bucket_entry, card_set, card_in_region, true /* within_howl */);
+    _coarsen_stats.record_coarsening(card_set_type(card_set) + G1CardSetCoarsenStats::CoarsenHowlOffset, !coarsened);
+    if (coarsened) {
+      // We have been the one coarsening this card set (and in the process added that card).
+      add_result = Added;
+      to_transfer = card_set;
+      break;
+    }
+    // Somebody else beat us to coarsening. Retry.
+    release_and_maybe_free_card_set(card_set);
+  }
+
+  if (increment_total && add_result == Added) {
+    Atomic::inc(&howl->_num_entries, memory_order_relaxed);
+  }
+
+  if (to_transfer != nullptr) {
+    transfer_cards_in_howl(parent_card_set, to_transfer, card_region);
+  }
+
+  release_and_maybe_free_card_set(card_set);
+  return add_result;
+}
+
+G1AddCardResult G1CardSet::add_to_bitmap(CardSetPtr card_set, uint card_in_region) {
+  G1CardSetBitMap* bitmap = card_set_ptr<G1CardSetBitMap>(card_set);
+  uint card_offset = _config->howl_bitmap_offset(card_in_region);
+  return bitmap->add(card_offset, _config->cards_in_howl_bitmap_threshold(), _config->num_cards_in_howl_bitmap());
+}
+
+G1AddCardResult G1CardSet::add_to_inline_ptr(CardSetPtr volatile* card_set_addr, CardSetPtr card_set, uint card_in_region) {
+  G1CardSetInlinePtr value(card_set_addr, card_set);
+  return value.add(card_in_region, _config->inline_ptr_bits_per_card(), _config->num_cards_in_inline_ptr());
+}
+
+G1CardSet::CardSetPtr G1CardSet::create_coarsened_array_of_cards(uint card_in_region, bool within_howl) {
+  uint8_t* data = nullptr;
+  CardSetPtr new_card_set;
+  if (within_howl) {
+    uint const size_in_bits = _config->num_cards_in_howl_bitmap();
+    uint card_offset = _config->howl_bitmap_offset(card_in_region);
+    data = allocate_mem_object(CardSetBitMap);
+    new (data) G1CardSetBitMap(card_offset, size_in_bits);
+    new_card_set = make_card_set_ptr(data, CardSetBitMap);
+  } else {
+    data = allocate_mem_object(CardSetHowl);
+    new (data) G1CardSetHowl(card_in_region, _config);
+    new_card_set = make_card_set_ptr(data, CardSetHowl);
+  }
+  return new_card_set;
+}
+
+bool G1CardSet::coarsen_card_set(volatile CardSetPtr* card_set_addr,
+                                 CardSetPtr cur_card_set,
+                                 uint card_in_region,
+                                 bool within_howl) {
+  CardSetPtr new_card_set = nullptr;
+
+  switch (card_set_type(cur_card_set)) {
+    case CardSetArrayOfCards : {
+      new_card_set = create_coarsened_array_of_cards(card_in_region, within_howl);
+      break;
+    }
+    case CardSetBitMap: {
+      new_card_set = FullCardSet;
+      break;
+    }
+    case CardSetInlinePtr: {
+      uint const size = _config->num_cards_in_array();
+      uint8_t* data = allocate_mem_object(CardSetArrayOfCards);
+      new (data) G1CardSetArray(card_in_region, size);
+      new_card_set = make_card_set_ptr(data, CardSetArrayOfCards);
+      break;
+    }
+    case CardSetHowl: {
+      new_card_set = FullCardSet; // anything will do at this point.
+      break;
+    }
+    default:
+      ShouldNotReachHere();
+  }
+
+  CardSetPtr old_value = Atomic::cmpxchg(card_set_addr, cur_card_set, new_card_set); // Memory order?
+  if (old_value == cur_card_set) {
+    // Success. Indicate that the cards from the current card set must be transferred
+    // by this caller.
+    // Release the hash table reference to the card. The caller still holds the
+    // reference to this card set, so it can never be released (and we do not need to
+    // check its result).
+    bool should_free = release_card_set(cur_card_set);
+    assert(!should_free, "must have had more than one reference");
+    // Free containers if cur_card_set is CardSetHowl
+    if (card_set_type(cur_card_set) == CardSetHowl) {
+      G1ReleaseCardsets rel(this);
+      card_set_ptr<G1CardSetHowl>(cur_card_set)->iterate(rel, _config->num_buckets_in_howl());
+    }
+    return true;
+  } else {
+    // Somebody else beat us to coarsening that card set. Exit, but clean up first.
+    if (new_card_set != FullCardSet) {
+      assert(new_card_set != nullptr, "must not be");
+      release_and_must_free_card_set(new_card_set);
+    }
+    return false;
+  }
+}
+
+class G1TransferCard : public StackObj {
+  G1CardSet* _card_set;
+  uint _region_idx;
+public:
+  G1TransferCard(G1CardSet* card_set, uint region_idx) : _card_set(card_set), _region_idx(region_idx) { }
+
+  void operator ()(uint card_idx) {
+    _card_set->add_card(_region_idx, card_idx, false);
+  }
+};
+
+void G1CardSet::transfer_cards(G1CardSetHashTableValue* table_entry, CardSetPtr source_card_set, uint card_region) {
+  assert(source_card_set != FullCardSet, "Should not need to transfer from full");
+  // Need to transfer old entries unless there is a Full card set in place now, i.e.
+  // the old type has been CardSetBitMap. "Full" contains all elements anyway.
+  if (card_set_type(source_card_set) != CardSetHowl) {
+    G1TransferCard iter(this, card_region);
+    iterate_cards_during_transfer(source_card_set, iter);
+  } else {
+    assert(card_set_type(source_card_set) == CardSetHowl, "must be");
+    // Need to correct for that the Full remembered set occupies more cards than the
+    // AoCS before.
+    Atomic::add(&_num_occupied, _config->max_cards_in_region() - table_entry->_num_occupied, memory_order_relaxed);
+  }
+}
+
+void G1CardSet::transfer_cards_in_howl(CardSetPtr parent_card_set,
+                                                     CardSetPtr source_card_set,
+                                                     uint card_region) {
+  assert(card_set_type(parent_card_set) == CardSetHowl, "must be");
+  assert(source_card_set != FullCardSet, "Should not need to transfer from full");
+  // Need to transfer old entries unless there is a Full card set in place now, i.e.
+  // the old type has been CardSetBitMap.
+  if (card_set_type(source_card_set) != CardSetBitMap) {
+    // We only need to transfer from anything below CardSetBitMap.
+    G1TransferCard iter(this, card_region);
+    iterate_cards_during_transfer(source_card_set, iter);
+  } else {
+    uint diff = _config->num_cards_in_howl_bitmap() - card_set_ptr<G1CardSetBitMap>(source_card_set)->num_bits_set();
+
+    // Need to correct for that the Full remembered set occupies more cards than the
+    // bitmap before.
+    // We add 1 element less because the values will be incremented
+    // in G1CardSet::add_card for the current addition or where already incremented in
+    // G1CardSet::add_to_howl after coarsening.
+    diff -= 1;
+
+    G1CardSetHowl* howling_array = card_set_ptr<G1CardSetHowl>(parent_card_set);
+    Atomic::add(&howling_array->_num_entries, diff, memory_order_relaxed);
+
+    bool should_grow_table = false;
+    G1CardSetHashTableValue* table_entry = get_or_add_card_set(card_region, &should_grow_table);
+    Atomic::add(&table_entry->_num_occupied, diff, memory_order_relaxed);
+
+    Atomic::add(&_num_occupied, diff, memory_order_relaxed);
+  }
+}
+
+G1AddCardResult G1CardSet::add_to_card_set(volatile CardSetPtr* card_set_addr, CardSetPtr card_set,  uint card_region, uint card_in_region, bool increment_total) {
+  assert(card_set_addr != nullptr, "Cannot add to empty cardset");
+
+  G1AddCardResult add_result;
+
+  switch (card_set_type(card_set)) {
+    case CardSetInlinePtr: {
+      add_result = add_to_inline_ptr(card_set_addr, card_set, card_in_region);
+      break;
+    }
+    case CardSetArrayOfCards : {
+      add_result = add_to_array(card_set, card_in_region);
+      break;
+    }
+    case CardSetBitMap: {
+      add_result = add_to_bitmap(card_set, card_in_region);
+      break;
+    }
+    case CardSetHowl: {
+      assert(CardSetHowl == card_set_type(FullCardSet), "must be");
+      if (card_set == FullCardSet) {
+        return Found;
+      }
+      add_result = add_to_howl(card_set, card_region, card_in_region, increment_total);
+      break;
+    }
+    default:
+      ShouldNotReachHere();
+  }
+
+  return add_result;
+}
+
+G1CardSetHashTableValue* G1CardSet::get_or_add_card_set(uint card_region, bool* should_grow_table) {
+  return _table->get_or_add(card_region, should_grow_table);
+}
+
+G1CardSet::CardSetPtr G1CardSet::get_card_set(uint card_region) {
+  return _table->get(card_region);
+}
+
+G1AddCardResult G1CardSet::add_card(uint card_region, uint card_in_region, bool increment_total) {
+  G1AddCardResult add_result;
+  CardSetPtr to_transfer = nullptr;
+  CardSetPtr card_set;
+
+  bool should_grow_table = false;
+  G1CardSetHashTableValue* table_entry = get_or_add_card_set(card_region, &should_grow_table);
+  while (true) {
+    card_set = acquire_card_set(&table_entry->_card_set);
+    add_result = add_to_card_set(&table_entry->_card_set, card_set, card_region, card_in_region, increment_total);
+
+    if (add_result != Overflow) {
+      break;
+    }
+    // Card set has overflown. Coarsen or retry.
+    bool coarsened = coarsen_card_set(&table_entry->_card_set, card_set, card_in_region);
+    _coarsen_stats.record_coarsening(card_set_type(card_set), !coarsened);
+    if (coarsened) {
+      // We have been the one coarsening this card set (and in the process added that card).
+      add_result = Added;
+      to_transfer = card_set;
+      break;
+    }
+    // Somebody else beat us to coarsening. Retry.
+    release_and_maybe_free_card_set(card_set);
+  }
+
+  if (increment_total && add_result == Added) {
+    Atomic::inc(&table_entry->_num_occupied, memory_order_relaxed);
+    Atomic::inc(&_num_occupied, memory_order_relaxed);
+  }
+  if (should_grow_table) {
+    _table->grow();
+  }
+  if (to_transfer != nullptr) {
+    transfer_cards(table_entry, to_transfer, card_region);
+  }
+
+  release_and_maybe_free_card_set(card_set);
+
+  return add_result;
+}
+
+bool G1CardSet::contains_card(uint card_region, uint card_in_region) {
+  assert(card_in_region < _config->max_cards_in_region(),
+         "Card %u is beyond max %u", card_in_region, _config->max_cards_in_region());
+
+  // Protect the card set from reclamation.
+  GlobalCounter::CriticalSection cs(Thread::current());
+  CardSetPtr card_set = get_card_set(card_region);
+  if (card_set == nullptr) {
+    return false;
+  } else if (card_set == FullCardSet) {
+    // contains_card() is not a performance critical method so we do not hide that
+    // case in the switch below.
+    return true;
+  }
+
+  switch (card_set_type(card_set)) {
+    case CardSetInlinePtr: {
+      G1CardSetInlinePtr ptr(card_set);
+      return ptr.contains(card_in_region, _config->inline_ptr_bits_per_card());
+    }
+    case CardSetArrayOfCards :  return card_set_ptr<G1CardSetArray>(card_set)->contains(card_in_region);
+    case CardSetBitMap: return card_set_ptr<G1CardSetBitMap>(card_set)->contains(card_in_region, _config->num_cards_in_howl_bitmap());
+    case CardSetHowl: {
+      G1CardSetHowl* howling_array = card_set_ptr<G1CardSetHowl>(card_set);
+
+      return howling_array->contains(card_in_region, _config);
+    }
+  }
+  ShouldNotReachHere();
+  return false;
+}
+
+void G1CardSet::print_info(outputStream* st, uint card_region, uint card_in_region) {
+  CardSetPtr card_set = get_card_set(card_region);
+  if (card_set == nullptr) {
+    st->print("NULL card set");
+    return;
+  } else if (card_set == FullCardSet) {
+    st->print("FULL card set)");
+    return;
+  }
+  switch (card_set_type(card_set)) {
+    case CardSetInlinePtr: {
+      st->print("InlinePtr not containing %u", card_in_region);
+      break;
+    }
+    case CardSetArrayOfCards :  {
+      st->print("AoC not containing %u", card_in_region);
+      break;
+    }
+    case CardSetBitMap: {
+      st->print("BitMap not containing %u", card_in_region);
+      break;
+    }
+    case CardSetHowl: {
+      st->print("CardSetHowl not containing %u", card_in_region);
+      break;
+    }
+    default: st->print("Unknown card set type %u", card_set_type(card_set)); ShouldNotReachHere(); break;
+  }
+}
+
+template <class CardVisitor>
+void G1CardSet::iterate_cards_during_transfer(CardSetPtr const card_set, CardVisitor& found) {
+  uint type = card_set_type(card_set);
+  assert(type == CardSetInlinePtr || type == CardSetArrayOfCards,
+         "invalid card set type %d to transfer from",
+         card_set_type(card_set));
+
+  switch (type) {
+    case CardSetInlinePtr: {
+      G1CardSetInlinePtr ptr(card_set);
+      ptr.iterate(found, _config->inline_ptr_bits_per_card());
+      return;
+    }
+    case CardSetArrayOfCards : {
+      card_set_ptr<G1CardSetArray>(card_set)->iterate(found);
+      return;
+    }
+    default:
+      ShouldNotReachHere();
+  }
+}
+
+void G1CardSet::iterate_containers(G1CardSetPtrIterator* found, bool at_safepoint) {
+  if (at_safepoint) {
+    _table->iterate_safepoint(found);
+  } else {
+    _table->iterate(found);
+  }
+}
+
+template <typename Closure>
+class G1ContainerCards {
+  Closure& _iter;
+  uint _region_idx;
+
+public:
+  G1ContainerCards(Closure& iter, uint region_idx) : _iter(iter), _region_idx(region_idx) { }
+
+  bool start_iterate(uint tag) { return true; }
+
+  void operator()(uint card_idx) {
+    _iter.do_card(_region_idx, card_idx);
+  }
+
+  void operator()(uint card_idx, uint length) {
+    for (uint i = 0; i < length; i++) {
+      _iter.do_card(_region_idx, card_idx);
+    }
+  }
+};
+
+void G1CardSet::iterate_cards(G1CardSetCardIterator& iter) {
+  G1CardSetMergeCardIterator<G1CardSetCardIterator, G1ContainerCards> cl(this, iter);
+  iterate_containers(&cl);
+}
+
+bool G1CardSet::occupancy_less_or_equal_to(size_t limit) const {
+  return occupied() <= limit;
+}
+
+bool G1CardSet::is_empty() const {
+  return _num_occupied == 0;
+}
+
+size_t G1CardSet::occupied() const {
+  return _num_occupied;
+}
+
+size_t G1CardSet::num_containers() {
+  class GetNumberOfContainers : public G1CardSetPtrIterator {
+  public:
+    size_t _count;
+
+    GetNumberOfContainers() : G1CardSetPtrIterator(), _count(0) { }
+
+    void do_cardsetptr(uint region_idx, size_t num_occupied, CardSetPtr card_set) override {
+      _count++;
+    }
+  } cl;
+
+  iterate_containers(&cl);
+  return cl._count;
+}
+
+G1CardSetCoarsenStats G1CardSet::coarsen_stats() {
+  return _coarsen_stats;
+}
+
+void G1CardSet::print_coarsen_stats(outputStream* out) {
+  _last_coarsen_stats.subtract_from(_coarsen_stats);
+  out->print("Coarsening (recent): ");
+  _last_coarsen_stats.print_on(out);
+  out->print("Coarsening (all): ");
+  _coarsen_stats.print_on(out);
+}
+
+size_t G1CardSet::mem_size() const {
+  return sizeof(*this) +
+         _table->mem_size() +
+         _mm->mem_size();
+}
+
+size_t G1CardSet::wasted_mem_size() const {
+  return _mm->wasted_mem_size();
+}
+
+size_t G1CardSet::static_mem_size() {
+  return sizeof(FullCardSet) + sizeof(_coarsen_stats);
+}
+
+void G1CardSet::clear() {
+  _table->reset();
+  _num_occupied = 0;
+  _mm->flush();
+}
+
+void G1CardSet::print(outputStream* os) {
+  _table->print(os);
+  _mm->print(os);
+}

src/hotspot/share/gc/g1/g1CardSet.hpp

@@ -0,0 +1,376 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1CARDSET_HPP
+#define SHARE_GC_G1_G1CARDSET_HPP
+
+#include "memory/allocation.hpp"
+#include "memory/padded.hpp"
+#include "oops/oopsHierarchy.hpp"
+#include "utilities/concurrentHashTable.hpp"
+#include "utilities/lockFreeStack.hpp"
+
+class G1CardSetAllocOptions;
+class G1CardSetBufferList;
+class G1CardSetHashTable;
+class G1CardSetHashTableValue;
+class G1CardSetMemoryManager;
+class Mutex;
+
+// The result of an attempt to add a card to that card set.
+enum G1AddCardResult {
+  Overflow,  // The card set is more than full. The entry may have been added. Need
+             // Coarsen and retry.
+  Found,     // The card is already in the set.
+  Added      // The card has been added to the set by this attempt.
+};
+
+class G1CardSetConfiguration {
+  uint _inline_ptr_bits_per_card;
+  uint _num_cards_in_array;
+  uint _num_cards_in_howl_bitmap;
+  uint _num_buckets_in_howl;
+  uint _max_cards_in_card_set;
+  uint _cards_in_howl_threshold;
+  uint _cards_in_howl_bitmap_threshold;
+  uint _log2_num_cards_in_howl_bitmap;
+  size_t _bitmap_hash_mask;
+
+  void log_configuration();
+public:
+
+  // Initialize card set configuration from globals.
+  G1CardSetConfiguration();
+  // Initialize card set configuration from parameters.
+  G1CardSetConfiguration(uint inline_ptr_bits_per_card,
+                         uint num_cards_in_array,
+                         double cards_in_bitmap_threshold,
+                         uint max_buckets_in_howl,
+                         double cards_in_howl_threshold,
+                         uint max_cards_in_cardset);
+
+  // Inline pointer configuration
+  uint inline_ptr_bits_per_card() const { return _inline_ptr_bits_per_card; }
+  uint num_cards_in_inline_ptr() const;
+  static uint num_cards_in_inline_ptr(uint bits_per_card);
+
+  // Array of Cards configuration
+  bool use_cards_in_array() const { return _num_cards_in_array != 0; } // Unused for now
+  // Number of cards in "Array of Cards" set; 0 to disable.
+  // Always coarsen to next level if full, so no specific threshold.
+  uint num_cards_in_array() const { return _num_cards_in_array; }
+
+  // Bitmap within Howl card set container configuration
+  bool use_cards_in_howl_bitmap() const { return _num_cards_in_howl_bitmap != 0; } // Unused for now
+  uint num_cards_in_howl_bitmap() const { return _num_cards_in_howl_bitmap; }
+  // (Approximate) Number of cards in bitmap to coarsen Howl Bitmap to Howl Full.
+  uint cards_in_howl_bitmap_threshold() const { return _cards_in_howl_bitmap_threshold; }
+  uint log2_num_cards_in_howl_bitmap() const {return _log2_num_cards_in_howl_bitmap;}
+
+  // Howl card set container configuration
+  uint num_buckets_in_howl() const { return _num_buckets_in_howl; }
+  // Threshold at which to turn howling arrays into Full.
+  uint cards_in_howl_threshold() const { return _cards_in_howl_threshold; }
+  uint howl_bitmap_offset(uint card_idx) const { return card_idx & _bitmap_hash_mask; }
+  // Given a card index, return the bucket in the array of card sets.
+  uint howl_bucket_index(uint card_idx) { return card_idx >> _log2_num_cards_in_howl_bitmap; }
+
+  // Full card configuration
+  // Maximum number of cards in a non-full card set for a single region. Card sets
+  // with more entries per region are coarsened to Full.
+  uint max_cards_in_region() const { return _max_cards_in_card_set; }
+
+  // Memory object types configuration
+  // Number of distinctly sized memory objects on the card set heap.
+  // Currently contains CHT-Nodes, ArrayOfCards, BitMaps, Howl
+  static constexpr uint num_mem_object_types() { return 4; }
+  // Returns the memory allocation options for the memory objects on the card set heap. The returned
+  // array must be freed by the caller.
+  G1CardSetAllocOptions* mem_object_alloc_options();
+
+  // For a given memory object, get a descriptive name.
+  static const char* mem_object_type_name_str(uint index);
+};
+
+// Collects coarsening statistics: how many attempts of each kind and how many
+// failed due to a competing thread doing the coarsening first.
+class G1CardSetCoarsenStats {
+public:
+  // Number of entries in the statistics tables: since we index with the source
+  // cardset of the coarsening, this is the total number of combinations of
+  // card sets - 1.
+  static constexpr size_t NumCoarsenCategories = 7;
+  // Coarsening statistics for the possible CardSetPtr in the Howl card set
+  // start from this offset.
+  static constexpr size_t CoarsenHowlOffset = 4;
+
+private:
+  // Indices are "from" indices.
+  size_t _coarsen_from[NumCoarsenCategories];
+  size_t _coarsen_collision[NumCoarsenCategories];
+
+public:
+  G1CardSetCoarsenStats() { reset(); }
+
+  void reset();
+
+  void subtract_from(G1CardSetCoarsenStats& other);
+
+  // Record a coarsening for the given tag/category. Collision should be true if
+  // this coarsening lost the race to do the coarsening of that category.
+  void record_coarsening(uint tag, bool collision);
+
+  void print_on(outputStream* out);
+};
+
+// Sparse set of card indexes comprising a remembered set on the Java heap. Card
+// size is assumed to be card table card size.
+//
+// Technically it is implemented using a ConcurrentHashTable that stores a card
+// set container for every region containing at least one card.
+//
+// There are in total five different containers, encoded in the ConcurrentHashTable
+// node as CardSetPtr. A CardSetPtr may cover the whole region or just a part of
+// it.
+// See its description below for more information.
+class G1CardSet : public CHeapObj<mtGCCardSet> {
+  friend class G1CardSetTest;
+  friend class G1CardSetMtTestTask;
+
+  template <typename Closure, template <typename> class CardorRanges>
+  friend class G1CardSetMergeCardIterator;
+
+  friend class G1TransferCard;
+
+  friend class G1ReleaseCardsets;
+
+  static G1CardSetCoarsenStats _coarsen_stats; // Coarsening statistics since VM start.
+  static G1CardSetCoarsenStats _last_coarsen_stats; // Coarsening statistics at last GC.
+public:
+  // Two lower bits are used to encode the card storage types
+  static const uintptr_t CardSetPtrHeaderSize = 2;
+
+  // CardSetPtr represents the card storage type of a given covered area. It encodes
+  // a type in the LSBs, in addition to having a few significant values.
+  //
+  // Possible encodings:
+  //
+  // 0...00000 free               (Empty, should never happen)
+  // 1...11111 full               All card indexes in the whole area this CardSetPtr covers are part of this container.
+  // X...XXX00 inline-ptr-cards   A handful of card indexes covered by this CardSetPtr are encoded within the CardSetPtr.
+  // X...XXX01 array of cards     The container is a contiguous array of card indexes.
+  // X...XXX10 bitmap             The container uses a bitmap to determine whether a given index is part of this set.
+  // X...XXX11 howl               This is a card set container containing an array of CardSetPtr, with each CardSetPtr
+  //                              limited to a sub-range of the original range. Currently only one level of this
+  //                              container is supported.
+  typedef void* CardSetPtr;
+  // Coarsening happens in the order below:
+  // CardSetInlinePtr -> CardSetArrayOfCards -> CardSetHowl -> Full
+  // Corsening of containers inside the CardSetHowl happens in the order:
+  // CardSetInlinePtr -> CardSetArrayOfCards -> CardSetBitMap -> Full
+  static const uintptr_t CardSetInlinePtr      = 0x0;
+  static const uintptr_t CardSetArrayOfCards   = 0x1;
+  static const uintptr_t CardSetBitMap         = 0x2;
+  static const uintptr_t CardSetHowl           = 0x3;
+
+  // The special sentinel values
+  static constexpr CardSetPtr FreeCardSet = nullptr;
+  // Unfortunately we can't make (G1CardSet::CardSetPtr)-1 constexpr because
+  // reinterpret_casts are forbidden in constexprs. Use a regular static instead.
+  static CardSetPtr FullCardSet;
+
+  static const uintptr_t CardSetPtrTypeMask    = ((uintptr_t)1 << CardSetPtrHeaderSize) - 1;
+
+  static CardSetPtr strip_card_set_type(CardSetPtr ptr) { return (CardSetPtr)((uintptr_t)ptr & ~CardSetPtrTypeMask); }
+
+  static uint card_set_type(CardSetPtr ptr) { return (uintptr_t)ptr & CardSetPtrTypeMask; }
+
+  template <class T>
+  static T* card_set_ptr(CardSetPtr ptr);
+
+private:
+  G1CardSetMemoryManager* _mm;
+  G1CardSetConfiguration* _config;
+
+  G1CardSetHashTable* _table;
+
+  // Total number of cards in this card set. This is a best-effort value, i.e. there may
+  // be (slightly) more cards in the card set than this value in reality.
+  size_t _num_occupied;
+
+  CardSetPtr make_card_set_ptr(void* value, uintptr_t type);
+
+  CardSetPtr acquire_card_set(CardSetPtr volatile* card_set_addr);
+  // Returns true if the card set should be released
+  bool release_card_set(CardSetPtr card_set);
+  // Release card set and free if needed.
+  void release_and_maybe_free_card_set(CardSetPtr card_set);
+  // Release card set and free (and it must be freeable).
+  void release_and_must_free_card_set(CardSetPtr card_set);
+
+  // Coarsens the CardSet cur_card_set to the next level; tries to replace the
+  // previous CardSet with a new one which includes the given card_in_region.
+  // coarsen_card_set does not transfer cards from cur_card_set
+  // to the new card_set. Transfer is achieved by transfer_cards.
+  // Returns true if this was the thread that coarsened the CardSet (and added the card).
+  bool coarsen_card_set(CardSetPtr volatile* card_set_addr,
+                        CardSetPtr cur_card_set,
+                        uint card_in_region, bool within_howl = false);
+
+  CardSetPtr create_coarsened_array_of_cards(uint card_in_region, bool within_howl);
+
+  // Transfer entries from source_card_set to a recently installed coarser storage type
+  // We only need to transfer anything finer than CardSetBitMap. "Full" contains
+  // all elements anyway.
+  void transfer_cards(G1CardSetHashTableValue* table_entry, CardSetPtr source_card_set, uint card_region);
+  void transfer_cards_in_howl(CardSetPtr parent_card_set, CardSetPtr source_card_set, uint card_region);
+
+  G1AddCardResult add_to_card_set(CardSetPtr volatile* card_set_addr, CardSetPtr card_set, uint card_region, uint card, bool increment_total = true);
+
+  G1AddCardResult add_to_inline_ptr(CardSetPtr volatile* card_set_addr, CardSetPtr card_set, uint card_in_region);
+  G1AddCardResult add_to_array(CardSetPtr card_set, uint card_in_region);
+  G1AddCardResult add_to_bitmap(CardSetPtr card_set, uint card_in_region);
+  G1AddCardResult add_to_howl(CardSetPtr parent_card_set, uint card_region, uint card_in_region, bool increment_total = true);
+
+  G1CardSetHashTableValue* get_or_add_card_set(uint card_region, bool* should_grow_table);
+  CardSetPtr get_card_set(uint card_region);
+
+  // Iterate over cards of a card set container during transfer of the cards from
+  // one container to another. Executes
+  //
+  //     void operator ()(uint card_idx)
+  //
+  // on the given class.
+  template <class CardVisitor>
+  void iterate_cards_during_transfer(CardSetPtr const card_set, CardVisitor& found);
+
+  // Iterate over the container, calling a method on every card or card range contained
+  // in the card container.
+  // For every container, first calls
+  //
+  //   void start_iterate(uint tag, uint region_idx);
+  //
+  // Then for every card or card range it calls
+  //
+  //   void do_card(uint card_idx);
+  //   void do_card_range(uint card_idx, uint length);
+  //
+  // where card_idx is the card index within that region_idx passed before in
+  // start_iterate().
+  //
+  template <class CardOrRangeVisitor>
+  void iterate_cards_or_ranges_in_container(CardSetPtr const card_set, CardOrRangeVisitor& found);
+
+  uint card_set_type_to_mem_object_type(uintptr_t type) const;
+  uint8_t* allocate_mem_object(uintptr_t type);
+  void free_mem_object(CardSetPtr card_set);
+
+public:
+  G1CardSetConfiguration* config() const { return _config; }
+
+  // Create a new remembered set for a particular heap region.
+  G1CardSet(G1CardSetConfiguration* config, G1CardSetMemoryManager* mm);
+  virtual ~G1CardSet();
+
+  // Adds the given card to this set, returning an appropriate result. If added,
+  // updates the total count.
+  G1AddCardResult add_card(uint card_region, uint card_in_region, bool increment_total = true);
+
+  bool contains_card(uint card_region, uint card_in_region);
+
+  void print_info(outputStream* st, uint card_region, uint card_in_region);
+
+  // Returns whether this remembered set (and all sub-sets) have an occupancy
+  // that is less or equal to the given occupancy.
+  bool occupancy_less_or_equal_to(size_t limit) const;
+
+  // Returns whether this remembered set (and all sub-sets) does not contain any entry.
+  bool is_empty() const;
+
+  // Returns the number of cards contained in this remembered set.
+  size_t occupied() const;
+
+  size_t num_containers();
+
+  static G1CardSetCoarsenStats coarsen_stats();
+  static void print_coarsen_stats(outputStream* out);
+
+  // Returns size of the actual remembered set containers in bytes.
+  size_t mem_size() const;
+  size_t wasted_mem_size() const;
+  // Returns the size of static data in bytes.
+  static size_t static_mem_size();
+
+  // Clear the entire contents of this remembered set.
+  void clear();
+
+  void print(outputStream* os);
+
+  // Various iterators - should be made inlineable somehow.
+  class G1CardSetPtrIterator {
+  public:
+    virtual void do_cardsetptr(uint region_idx, size_t num_occupied, CardSetPtr card_set) = 0;
+  };
+
+  void iterate_containers(G1CardSetPtrIterator* iter, bool safepoint = false);
+
+  class G1CardSetCardIterator {
+  public:
+    virtual void do_card(uint region_idx, uint card_idx) = 0;
+  };
+
+  void iterate_cards(G1CardSetCardIterator& iter);
+
+  // Iterate all cards for card set merging. Must be a CardOrRangeVisitor as
+  // explained above.
+  template <class CardOrRangeVisitor>
+  void iterate_for_merge(CardOrRangeVisitor& cl);
+};
+
+class G1CardSetHashTableValue {
+public:
+  using CardSetPtr = G1CardSet::CardSetPtr;
+
+  const uint _region_idx;
+  uint volatile _num_occupied;
+  CardSetPtr volatile _card_set;
+
+  G1CardSetHashTableValue(uint region_idx, CardSetPtr card_set) : _region_idx(region_idx), _num_occupied(0), _card_set(card_set) { }
+};
+
+class G1CardSetHashTableConfig : public StackObj {
+public:
+  using Value = G1CardSetHashTableValue;
+
+  static uintx get_hash(Value const& value, bool* is_dead) {
+    *is_dead = false;
+    return value._region_idx;
+  }
+  static void* allocate_node(void* context, size_t size, Value const& value);
+  static void free_node(void* context, void* memory, Value const& value);
+};
+
+typedef ConcurrentHashTable<G1CardSetHashTableConfig, mtGCCardSet> CardSetHash;
+
+#endif // SHARE_GC_G1_G1CARDSET_HPP

src/hotspot/share/gc/g1/g1CardSet.inline.hpp

@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1CARDSET_INLINE_HPP
+#define SHARE_GC_G1_G1CARDSET_INLINE_HPP
+
+#include "gc/g1/g1CardSet.hpp"
+#include "gc/g1/g1CardSetContainers.inline.hpp"
+#include "gc/g1/g1GCPhaseTimes.hpp"
+#include "runtime/atomic.hpp"
+#include "logging/log.hpp"
+
+template <class T>
+inline T* G1CardSet::card_set_ptr(CardSetPtr ptr) {
+  return (T*)strip_card_set_type(ptr);
+}
+
+inline G1CardSet::CardSetPtr G1CardSet::make_card_set_ptr(void* value, uintptr_t type) {
+  assert(card_set_type(value) == 0, "Given ptr " PTR_FORMAT " already has type bits set", p2i(value));
+  return (CardSetPtr)((uintptr_t)value | type);
+}
+
+template <class CardOrRangeVisitor>
+inline void G1CardSet::iterate_cards_or_ranges_in_container(CardSetPtr const card_set, CardOrRangeVisitor& found) {
+  switch (card_set_type(card_set)) {
+    case CardSetInlinePtr: {
+      if (found.start_iterate(G1GCPhaseTimes::MergeRSMergedInline)) {
+        G1CardSetInlinePtr ptr(card_set);
+        ptr.iterate(found, _config->inline_ptr_bits_per_card());
+      }
+      return;
+    }
+    case CardSetArrayOfCards : {
+      if (found.start_iterate(G1GCPhaseTimes::MergeRSMergedArrayOfCards)) {
+        card_set_ptr<G1CardSetArray>(card_set)->iterate(found);
+      }
+      return;
+    }
+    case CardSetBitMap: {
+      // There is no first-level bitmap spanning the whole area.
+      ShouldNotReachHere();
+      return;
+    }
+    case CardSetHowl: {
+      assert(card_set_type(FullCardSet) == CardSetHowl, "Must be");
+      if (card_set == FullCardSet) {
+        if (found.start_iterate(G1GCPhaseTimes::MergeRSMergedFull)) {
+          found(0, _config->max_cards_in_region());
+        }
+        return;
+      }
+      if (found.start_iterate(G1GCPhaseTimes::MergeRSMergedHowl)) {
+        card_set_ptr<G1CardSetHowl>(card_set)->iterate(found, _config);
+      }
+      return;
+    }
+  }
+  log_error(gc)("Unkown card set type %u", card_set_type(card_set));
+  ShouldNotReachHere();
+}
+
+template <typename Closure>
+class G1ContainerCardsOrRanges {
+  Closure& _iter;
+  uint _region_idx;
+
+public:
+  G1ContainerCardsOrRanges(Closure& iter, uint region_idx) : _iter(iter), _region_idx(region_idx) { }
+
+  bool start_iterate(uint tag) {
+    return _iter.start_iterate(tag, _region_idx);
+  }
+
+  void operator()(uint card_idx) {
+    _iter.do_card(card_idx);
+  }
+
+  void operator()(uint card_idx, uint length) {
+    _iter.do_card_range(card_idx, length);
+  }
+};
+
+template <typename Closure, template <typename> class CardOrRanges>
+class G1CardSetMergeCardIterator : public G1CardSet::G1CardSetPtrIterator {
+  G1CardSet* _card_set;
+  Closure& _iter;
+
+public:
+
+  G1CardSetMergeCardIterator(G1CardSet* card_set, Closure& iter) : _card_set(card_set), _iter(iter) { }
+
+  void do_cardsetptr(uint region_idx, size_t num_occupied, G1CardSet::CardSetPtr card_set) override {
+    CardOrRanges<Closure> cl(_iter, region_idx);
+    _card_set->iterate_cards_or_ranges_in_container(card_set, cl);
+  }
+};
+
+template <class CardOrRangeVisitor>
+inline void G1CardSet::iterate_for_merge(CardOrRangeVisitor& cl) {
+  G1CardSetMergeCardIterator<CardOrRangeVisitor, G1ContainerCardsOrRanges> cl2(this, cl);
+  iterate_containers(&cl2, true /* at_safepoint */);
+}
+
+#endif // SHARE_GC_G1_G1CARDSET_INLINE_HPP

src/hotspot/share/gc/g1/g1CardSetContainers.hpp

@@ -0,0 +1,294 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1CARDSETCONTAINERS_HPP
+#define SHARE_GC_G1_G1CARDSETCONTAINERS_HPP
+
+#include "gc/g1/g1CardSet.hpp"
+#include "memory/allocation.hpp"
+#include "runtime/atomic.hpp"
+#include "utilities/bitMap.inline.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/spinYield.hpp"
+
+#include "logging/log.hpp"
+
+#include "runtime/thread.inline.hpp"
+
+class G1CardSetInlinePtr : public StackObj {
+  friend class G1CardSetContainersTest;
+
+  typedef G1CardSet::CardSetPtr CardSetPtr;
+
+  CardSetPtr volatile * _value_addr;
+  CardSetPtr _value;
+
+  static const uint SizeFieldLen = 3;
+  static const uint SizeFieldPos = 2;
+  static const uint HeaderSize = G1CardSet::CardSetPtrHeaderSize + SizeFieldLen;
+
+  static const uint BitsInValue = sizeof(CardSetPtr) * BitsPerByte;
+
+  static const uintptr_t SizeFieldMask = (((uint)1 << SizeFieldLen) - 1) << SizeFieldPos;
+
+  static uint8_t card_pos_for(uint const idx, uint const bits_per_card) {
+    return (idx * bits_per_card + HeaderSize);
+  }
+
+  static CardSetPtr merge(CardSetPtr orig_value, uint card_in_region, uint idx, uint bits_per_card);
+
+  static uint card_at(CardSetPtr value, uint const idx, uint const bits_per_card) {
+    uint8_t card_pos = card_pos_for(idx, bits_per_card);
+    uint result = ((uintptr_t)value >> card_pos) & (((uintptr_t)1 << bits_per_card) - 1);
+    return result;
+  }
+public:
+  G1CardSetInlinePtr() : _value_addr(nullptr), _value((CardSetPtr)G1CardSet::CardSetInlinePtr) { }
+
+  G1CardSetInlinePtr(CardSetPtr value) : _value_addr(nullptr), _value(value) {
+    assert(((uintptr_t)_value & G1CardSet::CardSetInlinePtr) == G1CardSet::CardSetInlinePtr, "Value " PTR_FORMAT " is not a valid G1CardSetInPtr.", p2i(_value));
+  }
+
+  G1CardSetInlinePtr(CardSetPtr volatile* value_addr, CardSetPtr value) : _value_addr(value_addr), _value(value) {
+    assert(((uintptr_t)_value & G1CardSet::CardSetInlinePtr) == G1CardSet::CardSetInlinePtr, "Value " PTR_FORMAT " is not a valid G1CardSetInPtr.", p2i(_value));
+  }
+
+  G1AddCardResult add(uint const card_idx, uint const bits_per_card, uint const max_cards_in_inline_ptr);
+
+  bool contains(uint const card_idx, uint const bits_per_card);
+
+  template <class CardVisitor>
+  void iterate(CardVisitor& found, uint const bits_per_card);
+
+  operator CardSetPtr () { return _value; }
+
+  static uint max_cards_in_inline_ptr(uint bits_per_card) {
+    return (BitsInValue - HeaderSize) / bits_per_card;
+  }
+
+  static uint num_cards_in(CardSetPtr value) {
+    return ((uintptr_t)value & SizeFieldMask) >> SizeFieldPos;
+  }
+};
+
+
+// Common base class for card set containers where the memory for the entries is
+// managed on the (C-)heap. Depending on the current use, one of the two overlapping
+// members are used:
+//
+// While such an object is assigned to a card set container, we utilize the
+// reference count for memory management.
+//
+// In this case the object is one of three states:
+// 1: Live: The object is visible to other threads, thus can
+//    safely be accessed by other threads (_ref_count >= 3).
+// 2: Dead: The object is visible to only a single thread and may be
+//    safely reclaimed (_ref_count == 1).
+// 3: Reclaimed: The object's memory has been reclaimed ((_ref_count & 0x1) == 0).
+// To maintain these constraints, live objects should have ((_ref_count & 0x1) == 1),
+// which requires that we increment the reference counts by 2 starting at _ref_count = 3.
+//
+// When such an object is on a free list, we reuse the same field for linking
+// together those free objects.
+//
+// All but inline pointers are of this kind. For those, card entries are stored
+// directly in the CardSetPtr of the ConcurrentHashTable node.
+class G1CardSetContainer {
+private:
+  union {
+    G1CardSetContainer* _next;
+    uintptr_t _ref_count;
+  };
+
+public:
+  G1CardSetContainer() : _ref_count(3) { }
+
+  uintptr_t refcount() const { return Atomic::load_acquire(&_ref_count); }
+
+  bool try_increment_refcount();
+
+  // Decrement refcount potentially while racing increment, so we need
+  // to check the value after attempting to decrement.
+  uintptr_t decrement_refcount();
+
+  G1CardSetContainer* next() {
+    return _next;
+  }
+
+  G1CardSetContainer** next_addr() {
+    return &_next;
+  }
+
+  void set_next(G1CardSetContainer* next) {
+    _next = next;
+  }
+};
+
+class G1CardSetArray : public G1CardSetContainer {
+public:
+  typedef uint16_t EntryDataType;
+  typedef uint EntryCountType;
+  using CardSetPtr = G1CardSet::CardSetPtr;
+private:
+  EntryCountType _size;
+  EntryCountType volatile _num_entries;
+  EntryDataType _data[2];
+
+  static const EntryCountType LockBitMask = (EntryCountType)1 << (sizeof(EntryCountType) * BitsPerByte - 1);
+  static const EntryCountType EntryMask = LockBitMask - 1;
+
+  class G1CardSetArrayLocker : public StackObj {
+    EntryCountType volatile* _value;
+    EntryCountType volatile _original_value;
+    bool _success;
+  public:
+    G1CardSetArrayLocker(EntryCountType volatile* value);
+
+    EntryCountType num_entries() const { return _original_value; }
+    void inc_num_entries() { _success = true; }
+
+    ~G1CardSetArrayLocker() {
+      assert(((_original_value + _success) & EntryMask) == (EntryCountType)(_original_value + _success), "precondition!" );
+
+      Atomic::release_store(_value, (EntryCountType)(_original_value + _success));
+    }
+  };
+
+  template<typename Derived>
+  static size_t header_size_in_bytes_internal() {
+    return offset_of(Derived, _data);
+  }
+
+public:
+  G1CardSetArray(uint const card_in_region, EntryCountType num_elems);
+
+  G1AddCardResult add(uint card_idx);
+
+  bool contains(uint card_idx);
+
+  template <class CardVisitor>
+  void iterate(CardVisitor& found);
+
+  size_t num_entries() const { return _num_entries & EntryMask; }
+  size_t max_entries() const { return _size; }
+
+  static size_t header_size_in_bytes() { return header_size_in_bytes_internal<G1CardSetArray>(); }
+
+  static size_t size_in_bytes(size_t num_cards) {
+    return header_size_in_bytes() + sizeof(EntryDataType) * num_cards;
+  }
+};
+
+class G1CardSetBitMap : public G1CardSetContainer {
+  size_t _num_bits_set;
+  BitMap::bm_word_t _bits[1];
+
+  using CardSetPtr = G1CardSet::CardSetPtr;
+
+  template<typename Derived>
+  static size_t header_size_in_bytes_internal() {
+    return offset_of(Derived, _bits);
+  }
+
+public:
+  G1CardSetBitMap(uint const card_in_region, uint const size_in_bits);
+
+  G1AddCardResult add(uint card_idx, size_t threshold, size_t size_in_bits);
+
+  bool contains(uint card_idx, size_t size_in_bits) {
+    BitMapView bm(_bits, size_in_bits);
+    return bm.at(card_idx);
+  }
+
+  uint num_bits_set() const { return (uint)_num_bits_set; }
+
+  template <class CardVisitor>
+  void iterate(CardVisitor& found, size_t const size_in_bits, uint offset);
+
+  uint next(uint const idx, size_t const size_in_bits) {
+    BitMapView bm(_bits, size_in_bits);
+    return static_cast<uint>(bm.get_next_one_offset(idx));
+  }
+
+  static size_t header_size_in_bytes() { return header_size_in_bytes_internal<G1CardSetBitMap>(); }
+
+  static size_t size_in_bytes(size_t size_in_bits) { return header_size_in_bytes() + BitMap::calc_size_in_words(size_in_bits) * BytesPerWord; }
+};
+
+class G1CardSetHowl : public G1CardSetContainer {
+public:
+  typedef uint EntryCountType;
+  using CardSetPtr = G1CardSet::CardSetPtr;
+  EntryCountType volatile _num_entries;
+private:
+  CardSetPtr _buckets[2];
+  // Do not add class member variables beyond this point
+
+  template<typename Derived>
+  static size_t header_size_in_bytes_internal() {
+    return offset_of(Derived, _buckets);
+  }
+
+  // Iterates over the given CardSetPtr with at index in this Howl card set,
+  // applying a CardOrRangeVisitor on it.
+  template <class CardOrRangeVisitor>
+  void iterate_cardset(CardSetPtr const card_set, uint index, CardOrRangeVisitor& found, G1CardSetConfiguration* config);
+
+public:
+  G1CardSetHowl(EntryCountType card_in_region, G1CardSetConfiguration* config);
+
+  CardSetPtr* get_card_set_addr(EntryCountType index) {
+    return &_buckets[index];
+  }
+
+  bool contains(uint card_idx, G1CardSetConfiguration* config);
+
+  // Iterates over all CardSetPtrs in this Howl card set, applying a CardOrRangeVisitor
+  // on it.
+  template <class CardOrRangeVisitor>
+  void iterate(CardOrRangeVisitor& found, G1CardSetConfiguration* config);
+
+  // Iterates over all CardSetPtrs in this Howl card set. Calls
+  //
+  //   void operator ()(CardSetPtr* card_set_addr);
+  //
+  // on all of them.
+  template <class CardSetPtrVisitor>
+  void iterate(CardSetPtrVisitor& found, uint num_card_sets);
+
+  static EntryCountType num_buckets(size_t size_in_bits, size_t num_cards_in_array, size_t max_buckets);
+
+  static EntryCountType bitmap_size(size_t size_in_bits, uint num_buckets) {
+    EntryCountType num_cards = (EntryCountType)size_in_bits / num_buckets;
+    return round_up_power_of_2(num_cards);
+  }
+
+  static size_t header_size_in_bytes() { return header_size_in_bytes_internal<G1CardSetHowl>(); }
+
+  static size_t size_in_bytes(size_t num_arrays) {
+    return header_size_in_bytes() + sizeof(CardSetPtr) * num_arrays;
+  }
+};
+
+#endif // SHARE_GC_G1_G1CARDSETCONTAINERS_HPP

src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp

@@ -0,0 +1,341 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1CARDSETCONTAINERS_INLINE_HPP
+#define SHARE_GC_G1_G1CARDSETCONTAINERS_INLINE_HPP
+
+#include "gc/g1/g1CardSetContainers.hpp"
+#include "gc/g1/g1GCPhaseTimes.hpp"
+
+inline G1CardSetInlinePtr::CardSetPtr G1CardSetInlinePtr::merge(CardSetPtr orig_value, uint card_in_region, uint idx, uint bits_per_card) {
+  assert((idx & (SizeFieldMask >> SizeFieldPos)) == idx, "Index %u too large to fit into size field", idx);
+  assert(card_in_region < ((uint)1 << bits_per_card), "Card %u too large to fit into card value field", card_in_region);
+
+  uint8_t card_pos = card_pos_for(idx, bits_per_card);
+  assert(card_pos + bits_per_card < BitsInValue, "Putting card at pos %u with %u bits would extend beyond pointer", card_pos, bits_per_card);
+
+  // Check that we do not touch any fields we do not own.
+  uintptr_t mask = ((((uintptr_t)1 << bits_per_card) - 1) << card_pos);
+  assert(((uintptr_t)orig_value & mask) == 0, "The bits in the new range should be empty; orig_value " PTR_FORMAT " mask " PTR_FORMAT, p2i(orig_value), mask);
+
+  uintptr_t value = ((uintptr_t)(idx + 1) << SizeFieldPos) | ((uintptr_t)card_in_region << card_pos);
+  uintptr_t res = (((uintptr_t)orig_value & ~SizeFieldMask) | value);
+  return (CardSetPtr)res;
+}
+
+inline G1AddCardResult G1CardSetInlinePtr::add(uint card_idx, uint bits_per_card, uint max_cards_in_inline_ptr) {
+  assert(_value_addr != nullptr, "No value address available, cannot add to set.");
+
+  while (true) {
+    uint num_elems = num_cards_in(_value);
+    // Check if the card is already stored in the pointer.
+    if (contains(card_idx, bits_per_card)) {
+      return Found;
+    }
+    // Check if there is actually enough space.
+    if (num_elems >= max_cards_in_inline_ptr) {
+      return Overflow;
+    }
+    CardSetPtr new_value = merge(_value, card_idx, num_elems, bits_per_card);
+    CardSetPtr old_value = Atomic::cmpxchg(_value_addr, _value, new_value, memory_order_relaxed);
+    if (_value == old_value) {
+      return Added;
+    }
+    // Update values and retry.
+    _value = old_value;
+    // The value of the pointer may have changed to something different than
+    // an inline card set. Exit then instead of overwriting.
+    if (G1CardSet::card_set_type(_value) != G1CardSet::CardSetInlinePtr) {
+      return Overflow;
+    }
+  }
+}
+
+inline bool G1CardSetInlinePtr::contains(uint card_idx, uint bits_per_card) {
+  uint num_elems = num_cards_in(_value);
+  uintptr_t const card_mask = (1 << bits_per_card) - 1;
+
+  uintptr_t value = ((uintptr_t)_value) >> card_pos_for(0, bits_per_card);
+  // Check if the card is already stored in the pointer.
+  for (uint cur_idx = 0; cur_idx < num_elems; cur_idx++) {
+    if ((value & card_mask) == card_idx) {
+      return true;
+    }
+    value >>= bits_per_card;
+  }
+  return false;
+}
+
+template <class CardVisitor>
+inline void G1CardSetInlinePtr::iterate(CardVisitor& found, uint bits_per_card) {
+  uint const num_elems = num_cards_in(_value);
+  uintptr_t const card_mask = (1 << bits_per_card) - 1;
+
+  uintptr_t value = ((uintptr_t)_value) >> card_pos_for(0, bits_per_card);
+  for (uint cur_idx = 0; cur_idx < num_elems; cur_idx++) {
+    found(value & card_mask);
+    value >>= bits_per_card;
+  }
+}
+
+inline bool G1CardSetContainer::try_increment_refcount() {
+  uintptr_t old_value = refcount();
+  while (true) {
+    if (old_value < 3 || (old_value & 0x1) == 0) {  // reclaimed,  reference counts are odd numbers starting at 3
+      return false; // dead, can't revive.
+    }
+
+    uintptr_t new_value = old_value + 2;
+    uintptr_t ref_count = Atomic::cmpxchg(&_ref_count, old_value, new_value);
+    if (ref_count == old_value) {
+      return true;
+    }
+    old_value = ref_count;
+  }
+}
+
+inline uintptr_t G1CardSetContainer::decrement_refcount() {
+  uintptr_t old_value = refcount();
+  assert((old_value & 0x1) != 0 && old_value >= 3, "precondition");
+  return Atomic::sub(&_ref_count, 2u);
+}
+
+inline G1CardSetArray::G1CardSetArray(uint card_in_region, EntryCountType num_elems) :
+  G1CardSetContainer(),
+  _size(num_elems),
+  _num_entries(1) {
+  assert(_size > 0, "CardSetArray of size 0 not supported.");
+  assert(_size < LockBitMask, "Only support CardSetArray of size %u or smaller.", LockBitMask - 1);
+  _data[0] = card_in_region;
+}
+
+inline G1CardSetArray::G1CardSetArrayLocker::G1CardSetArrayLocker(EntryCountType volatile* value) :
+  _value(value),
+  _success(false) {
+  SpinYield s;
+  EntryCountType original_value = (*_value) & EntryMask;
+  while (true) {
+    EntryCountType old_value = Atomic::cmpxchg(_value,
+                                               original_value,
+                                               (EntryCountType)(original_value | LockBitMask));
+    if (old_value == original_value) {
+      // Succeeded locking the array.
+      _original_value = original_value;
+      break;
+    }
+    // Failed. Retry (with the lock bit stripped again).
+    original_value = old_value & EntryMask;
+    s.wait();
+  }
+}
+
+inline G1AddCardResult G1CardSetArray::add(uint card_idx) {
+  assert(card_idx < (1u << (sizeof(_data[0]) * BitsPerByte)),
+         "Card index %u does not fit card element.", card_idx);
+  EntryCountType num_entries = Atomic::load_acquire(&_num_entries) & EntryMask;
+  EntryCountType idx = 0;
+  for (; idx < num_entries; idx++) {
+    if (_data[idx] == card_idx) {
+      return Found;
+    }
+  }
+
+  // Since we did not find the card, lock.
+  G1CardSetArrayLocker x(&_num_entries);
+
+  // Reload number of entries from the G1CardSetArrayLocker as it might have changed.
+  // It already read the actual value with the necessary synchronization.
+  num_entries = x.num_entries();
+  // Look if the elements added while waiting for the lock are the same as our card.
+  for (; idx < num_entries; idx++) {
+    if (_data[idx] == card_idx) {
+      return Found;
+    }
+  }
+
+  // Check if there is space left.
+  if (num_entries == _size) {
+    return Overflow;
+  }
+
+  _data[num_entries] = card_idx;
+
+  x.inc_num_entries();
+
+  return Added;
+}
+
+inline bool G1CardSetArray::contains(uint card_idx) {
+  EntryCountType num_entries = Atomic::load_acquire(&_num_entries) & EntryMask;
+
+  for (EntryCountType idx = 0; idx < num_entries; idx++) {
+    if (_data[idx] == card_idx) {
+      return true;
+    }
+  }
+  return false;
+}
+
+template <class CardVisitor>
+void G1CardSetArray::iterate(CardVisitor& found) {
+  EntryCountType num_entries = Atomic::load_acquire(&_num_entries) & EntryMask;
+  for (EntryCountType idx = 0; idx < num_entries; idx++) {
+    found(_data[idx]);
+  }
+}
+
+inline G1CardSetBitMap::G1CardSetBitMap(uint card_in_region, uint size_in_bits) :
+  G1CardSetContainer(), _num_bits_set(1) {
+  assert(size_in_bits % (sizeof(_bits[0]) * BitsPerByte) == 0,
+         "Size %u should be aligned to bitmap word size.", size_in_bits);
+  BitMapView bm(_bits, size_in_bits);
+  bm.clear();
+  bm.set_bit(card_in_region);
+}
+
+inline G1AddCardResult G1CardSetBitMap::add(uint card_idx, size_t threshold, size_t size_in_bits) {
+  BitMapView bm(_bits, size_in_bits);
+  if (_num_bits_set >= threshold) {
+    return bm.at(card_idx) ? Found : Overflow;
+  }
+  if (bm.par_set_bit(card_idx)) {
+    Atomic::inc(&_num_bits_set, memory_order_relaxed);
+    return Added;
+  }
+  return Found;
+}
+
+template <class CardVisitor>
+inline void G1CardSetBitMap::iterate(CardVisitor& found, size_t size_in_bits, uint offset) {
+  BitMapView bm(_bits, size_in_bits);
+  BitMap::idx_t idx = bm.get_next_one_offset(0);
+  while (idx != size_in_bits) {
+    found((offset | (uint)idx));
+    idx = bm.get_next_one_offset(idx + 1);
+  }
+}
+
+inline G1CardSetHowl::G1CardSetHowl(EntryCountType card_in_region, G1CardSetConfiguration* config) :
+  G1CardSetContainer(),
+  _num_entries((config->num_cards_in_array() + 1)) /* Card Transfer will not increment _num_entries */ {
+  EntryCountType num_buckets = config->num_buckets_in_howl();
+  EntryCountType bucket = config->howl_bucket_index(card_in_region);
+  for (uint i = 0; i < num_buckets; ++i) {
+    _buckets[i] = G1CardSetInlinePtr();
+    if (i == bucket) {
+      G1CardSetInlinePtr value(&_buckets[i], _buckets[i]);
+      value.add(card_in_region, config->inline_ptr_bits_per_card(), config->num_cards_in_inline_ptr());
+    }
+  }
+}
+
+inline bool G1CardSetHowl::contains(uint card_idx, G1CardSetConfiguration* config) {
+  EntryCountType bucket = config->howl_bucket_index(card_idx);
+  CardSetPtr* array_entry = get_card_set_addr(bucket);
+  CardSetPtr card_set = Atomic::load_acquire(array_entry);
+
+  switch (G1CardSet::card_set_type(card_set)) {
+    case G1CardSet::CardSetArrayOfCards : {
+      return G1CardSet::card_set_ptr<G1CardSetArray>(card_set)->contains(card_idx);
+    }
+    case G1CardSet::CardSetBitMap: {
+      uint card_offset = config->howl_bitmap_offset(card_idx);
+      return G1CardSet::card_set_ptr<G1CardSetBitMap>(card_set)->contains(card_offset, config->num_cards_in_howl_bitmap());
+    }
+    case G1CardSet::CardSetInlinePtr: {
+      G1CardSetInlinePtr ptr(card_set);
+      return ptr.contains(card_idx, config->inline_ptr_bits_per_card());
+    }
+    case G1CardSet::CardSetHowl: {// Fullcard set entry
+      assert(card_set == G1CardSet::FullCardSet, "Must be");
+      return true;
+    }
+  }
+  return false;
+}
+
+template <class CardOrRangeVisitor>
+inline void G1CardSetHowl::iterate(CardOrRangeVisitor& found, G1CardSetConfiguration* config) {
+  for (uint i = 0; i < config->num_buckets_in_howl(); ++i) {
+    iterate_cardset(_buckets[i], i, found, config);
+  }
+}
+
+template <class CardSetPtrVisitor>
+inline void G1CardSetHowl::iterate(CardSetPtrVisitor& found, uint num_card_sets) {
+  for (uint i = 0; i < num_card_sets; ++i) {
+    found(&_buckets[i]);
+  }
+}
+
+template <class CardOrRangeVisitor>
+inline void G1CardSetHowl::iterate_cardset(CardSetPtr const card_set, uint index, CardOrRangeVisitor& found, G1CardSetConfiguration* config) {
+  switch (G1CardSet::card_set_type(card_set)) {
+    case G1CardSet::CardSetInlinePtr: {
+      if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlInline)) {
+        G1CardSetInlinePtr ptr(card_set);
+        ptr.iterate(found, config->inline_ptr_bits_per_card());
+      }
+      return;
+    }
+    case G1CardSet::CardSetArrayOfCards : {
+      if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlArrayOfCards)) {
+        G1CardSet::card_set_ptr<G1CardSetArray>(card_set)->iterate(found);
+      }
+      return;
+    }
+    case G1CardSet::CardSetBitMap: {
+      if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlBitmap)) {
+        uint offset = index << config->log2_num_cards_in_howl_bitmap();
+        G1CardSet::card_set_ptr<G1CardSetBitMap>(card_set)->iterate(found, config->num_cards_in_howl_bitmap(), offset);
+      }
+      return;
+    }
+    case G1CardSet::CardSetHowl: { // actually FullCardSet
+      if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlFull)) {
+        assert(card_set == G1CardSet::FullCardSet, "Must be");
+        uint offset = index << config->log2_num_cards_in_howl_bitmap();
+        for (uint i = 0; i < config->max_cards_in_region(); i++) {
+          found((offset | (uint)i));
+        }
+      }
+      return;
+    }
+  }
+}
+
+inline G1CardSetHowl::EntryCountType G1CardSetHowl::num_buckets(size_t size_in_bits, size_t num_cards_in_array, size_t max_num_buckets) {
+  size_t size_bitmap_bytes = BitMap::calc_size_in_words(size_in_bits) * BytesPerWord;
+  // Ensure that in the worst case arrays consume half the memory size
+  // of storing the entire bitmap
+  size_t max_size_arrays_bytes = size_bitmap_bytes / 2;
+  size_t size_array_bytes = num_cards_in_array * sizeof(G1CardSetArray::EntryDataType);
+  size_t num_arrays = max_size_arrays_bytes / size_array_bytes;
+  // We use shifts and masks for indexing the array. So round down to the next
+  // power of two to not use more than expected memory.
+  num_arrays = round_down_power_of_2(MAX2((size_t)1, MIN2(num_arrays, max_num_buckets)));
+  return (EntryCountType)num_arrays;
+}
+
+#endif // SHARE_GC_G1_G1CARDSETCONTAINERS_INLINE_HPP

src/hotspot/share/gc/g1/g1CardSetFreeMemoryTask.cpp

@@ -0,0 +1,203 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/g1/g1CardSetFreeMemoryTask.hpp"
+#include "gc/g1/g1CardSetMemory.inline.hpp"
+#include "gc/g1/g1CollectedHeap.hpp"
+#include "gc/g1/g1_globals.hpp"
+#include "gc/shared/gcTraceTime.inline.hpp"
+#include "gc/shared/suspendibleThreadSet.hpp"
+#include "heapRegionRemSet.hpp"
+#include "ci/ciUtilities.hpp"
+
+constexpr const char* G1CardSetFreeMemoryTask::_state_names[];
+
+const char* G1CardSetFreeMemoryTask::get_state_name(State value) const {
+  return _state_names[static_cast<std::underlying_type_t<State>>(value)];
+}
+
+bool G1CardSetFreeMemoryTask::deadline_exceeded(jlong deadline) {
+  return os::elapsed_counter() >= deadline;
+}
+
+static size_t keep_size(size_t free, size_t used, double percent) {
+  size_t to_keep = used * percent;
+  return MIN2(free, to_keep);
+}
+
+bool G1CardSetFreeMemoryTask::calculate_return_infos(jlong deadline) {
+  // Ignore the deadline in this step as it is very short.
+
+  G1CardSetMemoryStats used = _total_used;
+  G1CardSetMemoryStats free = G1CardSetFreePool::free_list_sizes();
+
+  _return_info = new G1ReturnMemoryProcessorSet(used.num_pools());
+  for (uint i = 0; i < used.num_pools(); i++) {
+    size_t return_to_vm_size = keep_size(free._num_mem_sizes[i],
+                                         used._num_mem_sizes[i],
+                                         G1RemSetFreeMemoryKeepExcessRatio);
+    log_trace(gc, task)("Card Set Free Memory: Type %s: Free: %zu (%zu) "
+                        "Used: %zu Keep: %zu",
+                        G1CardSetConfiguration::mem_object_type_name_str(i),
+                        free._num_mem_sizes[i], free._num_buffers[i],
+                        used._num_mem_sizes[i], return_to_vm_size);
+
+    _return_info->append(new G1ReturnMemoryProcessor(return_to_vm_size));
+  }
+
+  G1CardSetFreePool::update_unlink_processors(_return_info);
+  return false;
+}
+
+bool G1CardSetFreeMemoryTask::return_memory_to_vm(jlong deadline) {
+  for (int i = 0; i < _return_info->length(); i++) {
+    G1ReturnMemoryProcessor* info = _return_info->at(i);
+    if (!info->finished_return_to_vm()) {
+      if (info->return_to_vm(deadline)) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+bool G1CardSetFreeMemoryTask::return_memory_to_os(jlong deadline) {
+  for (int i = 0; i < _return_info->length(); i++) {
+    G1ReturnMemoryProcessor* info = _return_info->at(i);
+    if (!info->finished_return_to_os()) {
+      if (info->return_to_os(deadline)) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+bool G1CardSetFreeMemoryTask::cleanup_return_infos() {
+  for (int i = 0; i < _return_info->length(); i++) {
+     G1ReturnMemoryProcessor* info = _return_info->at(i);
+     delete info;
+  }
+  delete _return_info;
+
+  _return_info = nullptr;
+  return false;
+}
+
+bool G1CardSetFreeMemoryTask::free_excess_card_set_memory() {
+  jlong start = os::elapsed_counter();
+  jlong end = start +
+              (os::elapsed_frequency() / 1000) * G1RemSetFreeMemoryStepDurationMillis;
+
+  log_trace(gc, task)("Card Set Free Memory: Step start %1.3f end %1.3f",
+                      TimeHelper::counter_to_millis(start), TimeHelper::counter_to_millis(end));
+
+  State next_state;
+
+  do {
+    switch (_state) {
+      case State::CalculateUsed: {
+        if (calculate_return_infos(end)) {
+          next_state = _state;
+          return true;
+        }
+        next_state = State::ReturnToVM;
+        break;
+      }
+      case State::ReturnToVM: {
+        if (return_memory_to_vm(end)) {
+          next_state = _state;
+          return true;
+        }
+        next_state = State::ReturnToOS;
+        break;
+      }
+      case State::ReturnToOS: {
+        if (return_memory_to_os(end)) {
+          next_state = _state;
+          return true;
+        }
+        next_state = State::Cleanup;
+        break;
+      }
+      case State::Cleanup: {
+        cleanup_return_infos();
+        next_state = State::Inactive;
+        break;
+      }
+      default:
+        log_error(gc, task)("Should not try to free excess card set memory in %s state", get_state_name(_state));
+        ShouldNotReachHere();
+        break;
+    }
+
+    set_state(next_state);
+  } while (_state != State::Inactive && !deadline_exceeded(end));
+
+  log_trace(gc, task)("Card Set Free Memory: Step took %1.3fms, done %s",
+                      TimeHelper::counter_to_millis(os::elapsed_counter() - start),
+                      bool_to_str(_state == State::CalculateUsed));
+
+  return is_active();
+}
+
+void G1CardSetFreeMemoryTask::set_state(State new_state) {
+  log_trace(gc, task)("Card Set Free Memory: State change from %s to %s",
+                      get_state_name(_state),
+                      get_state_name(new_state));
+  _state = new_state;
+}
+
+bool G1CardSetFreeMemoryTask::is_active() const {
+  return _state != State::Inactive;
+}
+
+jlong G1CardSetFreeMemoryTask::reschedule_delay_ms() const {
+  return G1RemSetFreeMemoryRescheduleDelayMillis;
+}
+
+G1CardSetFreeMemoryTask::G1CardSetFreeMemoryTask(const char* name) :
+  G1ServiceTask(name), _state(State::CalculateUsed), _return_info(nullptr) { }
+
+void G1CardSetFreeMemoryTask::execute() {
+  SuspendibleThreadSetJoiner sts;
+
+  if (free_excess_card_set_memory()) {
+    schedule(reschedule_delay_ms());
+  }
+}
+
+void G1CardSetFreeMemoryTask::notify_new_stats(G1CardSetMemoryStats* young_gen_stats,
+                                               G1CardSetMemoryStats* collection_set_candidate_stats) {
+  assert_at_safepoint_on_vm_thread();
+
+  _total_used = *young_gen_stats;
+  _total_used.add(*collection_set_candidate_stats);
+
+  if (!is_active()) {
+    set_state(State::CalculateUsed);
+    G1CollectedHeap::heap()->service_thread()->schedule_task(this, 0);
+  }
+}

src/hotspot/share/gc/g1/g1CardSetFreeMemoryTask.hpp

@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1CARDSETFREEMEMORYTASK_HPP
+#define SHARE_GC_G1_G1CARDSETFREEMEMORYTASK_HPP
+
+#include "gc/g1/g1ServiceThread.hpp"
+#include "gc/g1/g1CardSetMemory.hpp"
+#include "gc/g1/heapRegionRemSet.hpp"
+#include "utilities/growableArray.hpp"
+#include "utilities/ticks.hpp"
+
+class G1CardSetBuffer;
+
+// Task handling deallocation of free card set memory.
+class G1CardSetFreeMemoryTask : public G1ServiceTask {
+
+  enum class State : uint {
+    Inactive,
+    CalculateUsed,
+    ReturnToVM,
+    ReturnToOS,
+    Cleanup
+  };
+
+  static constexpr const char* _state_names[] = { "Invalid",
+                                                  "CalculateUsed",
+                                                  "ReturnToVM",
+                                                  "ReturnToOS",
+                                                  "Cleanup" };
+
+  const char* get_state_name(State value) const;
+
+  State _state;
+
+  // Current total card set memory usage.
+  G1CardSetMemoryStats _total_used;
+
+  typedef G1CardSetFreePool::G1ReturnMemoryProcessor G1ReturnMemoryProcessor;
+  typedef G1CardSetFreePool::G1ReturnMemoryProcessorSet G1ReturnMemoryProcessorSet;
+
+  G1ReturnMemoryProcessorSet* _return_info;
+
+  // Returns whether the given deadline has passed.
+  bool deadline_exceeded(jlong deadline);
+
+  // Methods for the tasks to be done. They all return true if that step has
+  // completed.
+  bool calculate_return_infos(jlong deadline);
+  bool return_memory_to_vm(jlong deadline);
+  bool return_memory_to_os(jlong deadline);
+  bool cleanup_return_infos();
+
+  // Free excess card set memory, main method. Returns true if there is more work
+  // to do.
+  bool free_excess_card_set_memory();
+
+  void set_state(State new_state);
+  // Returns whether we are currently processing a recent request.
+  bool is_active() const;
+
+  // The delay used to reschedule this task if not all work has been completed.
+  jlong reschedule_delay_ms() const;
+
+public:
+  explicit G1CardSetFreeMemoryTask(const char* name);
+
+  void execute() override;
+
+  // Notify the task of new used remembered set memory statistics for the young
+  // generation and the collection set candidate sets.
+  void notify_new_stats(G1CardSetMemoryStats* young_gen_stats,
+                        G1CardSetMemoryStats* collection_set_candidate_stats);
+};
+
+#endif // SHARE_GC_G1_G1CARDSETFREEMEMORYTASK_HPP

src/hotspot/share/gc/g1/g1CardSetMemory.cpp

@@ -0,0 +1,478 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/g1/g1CardSetMemory.inline.hpp"
+#include "logging/log.hpp"
+#include "runtime/atomic.hpp"
+#include "utilities/formatBuffer.hpp"
+#include "utilities/ostream.hpp"
+
+G1CardSetBuffer::G1CardSetBuffer(uint elem_size, uint num_instances, G1CardSetBuffer* next) :
+    _elem_size(elem_size), _num_elems(num_instances), _next(next), _next_allocate(0) {
+
+  _buffer = NEW_C_HEAP_ARRAY(char, (size_t)_num_elems * elem_size, mtGCCardSet);
+}
+
+G1CardSetBuffer::~G1CardSetBuffer() {
+  FREE_C_HEAP_ARRAY(mtGCCardSet, _buffer);
+}
+
+void* G1CardSetBuffer::get_new_buffer_elem() {
+  if (_next_allocate >= _num_elems) {
+    return nullptr;
+  }
+  uint result = Atomic::fetch_and_add(&_next_allocate, 1u, memory_order_relaxed);
+  if (result >= _num_elems) {
+    return nullptr;
+  }
+  void* r = _buffer + (uint)result * _elem_size;
+  return r;
+}
+
+void G1CardSetBufferList::bulk_add(G1CardSetBuffer& first, G1CardSetBuffer& last, size_t num, size_t mem_size) {
+  _list.prepend(first, last);
+  Atomic::add(&_num_buffers, num, memory_order_relaxed);
+  Atomic::add(&_mem_size, mem_size, memory_order_relaxed);
+}
+
+void G1CardSetBufferList::print_on(outputStream* out, const char* prefix) {
+  out->print_cr("%s: buffers %zu size %zu", prefix, Atomic::load(&_num_buffers), Atomic::load(&_mem_size));
+}
+
+G1CardSetBuffer* G1CardSetBufferList::get() {
+  GlobalCounter::CriticalSection cs(Thread::current());
+
+  G1CardSetBuffer* result = _list.pop();
+  if (result != nullptr) {
+    Atomic::dec(&_num_buffers, memory_order_relaxed);
+    Atomic::sub(&_mem_size, result->mem_size(), memory_order_relaxed);
+  }
+  return result;
+}
+
+G1CardSetBuffer* G1CardSetBufferList::get_all(size_t& num_buffers, size_t& mem_size) {
+  GlobalCounter::CriticalSection cs(Thread::current());
+
+  G1CardSetBuffer* result = _list.pop_all();
+  num_buffers = Atomic::load(&_num_buffers);
+  mem_size = Atomic::load(&_mem_size);
+
+  if (result != nullptr) {
+    Atomic::sub(&_num_buffers, num_buffers, memory_order_relaxed);
+    Atomic::sub(&_mem_size, mem_size, memory_order_relaxed);
+  }
+  return result;
+}
+
+void G1CardSetBufferList::free_all() {
+  size_t num_freed = 0;
+  size_t mem_size_freed = 0;
+  G1CardSetBuffer* cur;
+
+  while ((cur = _list.pop()) != nullptr) {
+    mem_size_freed += cur->mem_size();
+    num_freed++;
+    delete cur;
+  }
+
+  Atomic::sub(&_num_buffers, num_freed, memory_order_relaxed);
+  Atomic::sub(&_mem_size, mem_size_freed, memory_order_relaxed);
+}
+
+template <class Elem>
+G1CardSetAllocator<Elem>::G1CardSetAllocator(const char* name,
+                                             const G1CardSetAllocOptions& buffer_options,
+                                             G1CardSetBufferList* free_buffer_list) :
+  _alloc_options(buffer_options),
+  _first(nullptr),
+  _last(nullptr),
+  _num_buffers(0),
+  _mem_size(0),
+  _free_buffer_list(free_buffer_list),
+  _transfer_lock(false),
+  _free_nodes_list(),
+  _pending_nodes_list(),
+  _num_pending_nodes(0),
+  _num_free_nodes(0),
+  _num_allocated_nodes(0),
+  _num_available_nodes(0)
+{
+  assert(elem_size() >= sizeof(G1CardSetContainer), "Element instance size %u for allocator %s too small",
+         elem_size(), name);
+  assert(_free_buffer_list != nullptr, "precondition!");
+}
+
+template <class Elem>
+bool G1CardSetAllocator<Elem>::try_transfer_pending() {
+  // Attempt to claim the lock.
+  if (Atomic::load_acquire(&_transfer_lock) || // Skip CAS if likely to fail.
+      Atomic::cmpxchg(&_transfer_lock, false, true)) {
+    return false;
+  }
+  // Have the lock; perform the transfer.
+
+  // Claim all the pending nodes.
+  G1CardSetContainer* first = _pending_nodes_list.pop_all();
+
+  if (first != nullptr) {
+    // Prepare to add the claimed nodes, and update _num_pending_nodes.
+    G1CardSetContainer* last = first;
+    Atomic::load_acquire(&_num_pending_nodes);
+
+    uint count = 1;
+    for (G1CardSetContainer* next = first->next(); next != nullptr; next = next->next()) {
+      last = next;
+      ++count;
+    }
+
+    Atomic::sub(&_num_pending_nodes, count);
+
+    // Wait for any in-progress pops to avoid ABA for them.
+    GlobalCounter::write_synchronize();
+    // Add synchronized nodes to _free_node_list.
+    // Update count first so there can be no underflow in allocate().
+    Atomic::add(&_num_free_nodes, count);
+    _free_nodes_list.prepend(*first, *last);
+  }
+  Atomic::release_store(&_transfer_lock, false);
+  return true;
+}
+
+template <class Elem>
+void G1CardSetAllocator<Elem>::free(Elem* elem) {
+  assert(elem != nullptr, "precondition");
+  assert(elem_size() >= sizeof(G1CardSetContainer), "size mismatch");
+  // Desired minimum transfer batch size.  There is relatively little
+  // importance to the specific number.  It shouldn't be too big, else
+  // we're wasting space when the release rate is low.  If the release
+  // rate is high, we might accumulate more than this before being
+  // able to start a new transfer, but that's okay.  Also note that
+  // the allocation rate and the release rate are going to be fairly
+  // similar, due to how the buffers are used. - kbarret
+  uint const trigger_transfer = 10;
+
+  uint pending_count = Atomic::add(&_num_pending_nodes, 1u, memory_order_relaxed);
+
+  G1CardSetContainer* node =  reinterpret_cast<G1CardSetContainer*>(reinterpret_cast<char*>(elem));
+
+  node->set_next(nullptr);
+  assert(node->next() == nullptr, "precondition");
+
+  _pending_nodes_list.push(*node);
+
+  if (pending_count > trigger_transfer) {
+    try_transfer_pending();
+  }
+}
+
+template <class Elem>
+void G1CardSetAllocator<Elem>::drop_all() {
+  _free_nodes_list.pop_all();
+  _pending_nodes_list.pop_all();
+  G1CardSetBuffer* cur = Atomic::load_acquire(&_first);
+
+  if (cur != nullptr) {
+    assert(_last != nullptr, "If there is at least one element, there must be a last one.");
+
+    G1CardSetBuffer* first = cur;
+#ifdef ASSERT
+    // Check list consistency.
+    G1CardSetBuffer* last = cur;
+    uint num_buffers = 0;
+    size_t mem_size = 0;
+    while (cur != nullptr) {
+      mem_size += cur->mem_size();
+      num_buffers++;
+
+      G1CardSetBuffer* next = cur->next();
+      last = cur;
+      cur = next;
+    }
+#endif
+    assert(num_buffers == _num_buffers, "Buffer count inconsistent %u %u", num_buffers, _num_buffers);
+    assert(mem_size == _mem_size, "Memory size inconsistent");
+    assert(last == _last, "Inconsistent last element");
+
+    _free_buffer_list->bulk_add(*first, *_last, _num_buffers, _mem_size);
+  }
+
+  _first = nullptr;
+  _last = nullptr;
+  _num_available_nodes = 0;
+  _num_allocated_nodes = 0;
+  _num_pending_nodes = 0;
+  _num_buffers = 0;
+  _mem_size = 0;
+  _num_free_nodes = 0;
+}
+
+template <class Elem>
+void G1CardSetAllocator<Elem>::print(outputStream* os) {
+  os->print("MA " PTR_FORMAT ": %u elems pending (allocated %u available %u) used %.3f highest %u buffers %u size %zu ",
+                p2i(this), _num_pending_nodes, _num_allocated_nodes, _num_available_nodes, percent_of(_num_allocated_nodes - _num_pending_nodes, _num_available_nodes), _first != nullptr ? _first->num_elems() : 0, _num_buffers, mem_size());
+}
+
+G1CardSetMemoryStats::G1CardSetMemoryStats() {
+  clear();
+}
+
+G1CardSetMemoryStats::G1CardSetMemoryStats(void(*fn)(const void*,uint,size_t&,size_t&), const void* context) {
+  clear();
+  for (uint i = 0; i < num_pools(); i++) {
+    fn(context, i, _num_mem_sizes[i], _num_buffers[i]);
+  }
+}
+
+void G1CardSetMemoryStats::clear() {
+  for (uint i = 0; i < num_pools(); i++) {
+    _num_mem_sizes[i] = 0;
+    _num_buffers[i] = 0;
+  }
+}
+
+void G1CardSetFreePool::update_unlink_processors(G1ReturnMemoryProcessorSet* unlink_processor) {
+  uint num_free_lists = _freelist_pool.num_free_lists();
+
+  for (uint i = 0; i < num_free_lists; i++) {
+    unlink_processor->at(i)->visit_free_list(_freelist_pool.free_list(i));
+  }
+}
+
+void G1CardSetFreePool::G1ReturnMemoryProcessor::visit_free_list(G1CardSetBufferList* source) {
+  assert(_source == nullptr, "already visited");
+  if (_return_to_vm_size > 0) {
+    _source = source;
+  } else {
+    assert(_source == nullptr, "must be");
+  }
+  if (source->mem_size() > _return_to_vm_size) {
+    _first = source->get_all(_num_unlinked, _unlinked_bytes);
+  } else {
+    assert(_first == nullptr, "must be");
+  }
+  // Above we were racing with other threads getting the contents of the free list,
+  // so while we might have been asked to return something to the OS initially,
+  // the free list might be empty anyway. In this case just reset internal values
+  // used for checking whether there is work available.
+  if (_first == nullptr) {
+    _source = nullptr;
+    _return_to_vm_size = 0;
+  }
+}
+
+bool G1CardSetFreePool::G1ReturnMemoryProcessor::return_to_vm(jlong deadline) {
+  assert(!finished_return_to_vm(), "already returned everything to the VM");
+  assert(_first != nullptr, "must have element to return");
+
+  size_t keep_size = 0;
+  size_t keep_num = 0;
+
+  G1CardSetBuffer* cur = _first;
+  G1CardSetBuffer* last = nullptr;
+
+  while (cur != nullptr && _return_to_vm_size > 0) {
+    size_t cur_size = cur->mem_size();
+    _return_to_vm_size -= MIN2(_return_to_vm_size, cur_size);
+
+    keep_size += cur_size;
+    keep_num++;
+
+    last = cur;
+    cur = cur->next();
+    // To ensure progress, perform the deadline check here.
+    if (os::elapsed_counter() > deadline) {
+      break;
+    }
+  }
+
+  assert(_first != nullptr, "must be");
+  assert(last != nullptr, "must be");
+
+  last->set_next(nullptr);
+
+  // Wait for any in-progress pops to avoid ABA for them.
+  GlobalCounter::write_synchronize();
+  _source->bulk_add(*_first, *last, keep_num, keep_size);
+  _first = cur;
+
+  log_trace(gc, task)("Card Set Free Memory: Returned to VM %zu buffers size %zu", keep_num, keep_size);
+
+  // _return_to_vm_size may be larger than what is available in the list at the
+  // time we actually get the list. I.e. the list and _return_to_vm_size may be
+  // inconsistent.
+  // So also check if we actually already at the end of the list for the exit
+  // condition.
+  if (_return_to_vm_size == 0 || _first == nullptr) {
+    _source = nullptr;
+    _return_to_vm_size = 0;
+  }
+  return _source != nullptr;
+}
+
+bool G1CardSetFreePool::G1ReturnMemoryProcessor::return_to_os(jlong deadline) {
+  assert(finished_return_to_vm(), "not finished returning to VM");
+  assert(!finished_return_to_os(), "already returned everything to the OS");
+
+  // Now delete the rest.
+  size_t num_delete = 0;
+  size_t mem_size_deleted = 0;
+
+  while (_first != nullptr) {
+    G1CardSetBuffer* next = _first->next();
+    num_delete++;
+    mem_size_deleted += _first->mem_size();
+    delete _first;
+    _first = next;
+
+    // To ensure progress, perform the deadline check here.
+    if (os::elapsed_counter() > deadline) {
+      break;
+    }
+  }
+
+  log_trace(gc, task)("Card Set Free Memory: Return to OS %zu buffers size %zu", num_delete, mem_size_deleted);
+
+  return _first != nullptr;
+}
+
+G1CardSetFreePool G1CardSetFreePool::_freelist_pool(G1CardSetConfiguration::num_mem_object_types());
+
+G1CardSetFreePool::G1CardSetFreePool(uint num_free_lists) :
+  _num_free_lists(num_free_lists) {
+
+  _free_lists = NEW_C_HEAP_ARRAY(G1CardSetBufferList, _num_free_lists, mtGC);
+  for (uint i = 0; i < _num_free_lists; i++) {
+    new (&_free_lists[i]) G1CardSetBufferList();
+  }
+}
+
+G1CardSetFreePool::~G1CardSetFreePool() {
+  for (uint i = 0; i < _num_free_lists; i++) {
+    _free_lists[i].~G1CardSetBufferList();
+  }
+  FREE_C_HEAP_ARRAY(mtGC, _free_lists);
+}
+
+static void collect_mem_sizes(const void* context, uint i, size_t& mem_size, size_t& num_buffers) {
+  ((G1CardSetFreePool*)context)->get_size(i, mem_size, num_buffers);
+}
+
+void G1CardSetFreePool::get_size(uint i, size_t& mem_size, size_t& num_buffers) const {
+  mem_size = _free_lists[i].mem_size();
+  num_buffers = _free_lists[i].num_buffers();
+}
+
+G1CardSetMemoryStats G1CardSetFreePool::memory_sizes() const {
+  return G1CardSetMemoryStats(collect_mem_sizes, this);
+}
+
+size_t G1CardSetFreePool::mem_size() const {
+  size_t result = 0;
+  for (uint i = 0; i < _num_free_lists; i++) {
+    result += _free_lists[i].mem_size();
+  }
+  return result;
+}
+
+void G1CardSetFreePool::print_on(outputStream* out) {
+  out->print_cr("  Free Pool: size %zu", free_list_pool()->mem_size());
+  for (uint i = 0; i < _num_free_lists; i++) {
+    FormatBuffer<> fmt("    %s", G1CardSetConfiguration::mem_object_type_name_str(i));
+    _free_lists[i].print_on(out, fmt);
+  }
+}
+
+G1CardSetMemoryManager::G1CardSetMemoryManager(G1CardSetConfiguration* config,
+                                               G1CardSetFreePool* free_list_pool) : _config(config) {
+
+  _allocators = NEW_C_HEAP_ARRAY(G1CardSetAllocator<G1CardSetContainer>,
+                                 _config->num_mem_object_types(),
+                                 mtGC);
+  G1CardSetAllocOptions* alloc_options = _config->mem_object_alloc_options();
+  for (uint i = 0; i < num_mem_object_types(); i++) {
+    new (&_allocators[i]) G1CardSetAllocator<G1CardSetContainer>(_config->mem_object_type_name_str(i),
+                                                                 alloc_options[i],
+                                                                 free_list_pool->free_list(i));
+  }
+  FREE_C_HEAP_ARRAY(size_t, alloc_options);
+}
+
+uint G1CardSetMemoryManager::num_mem_object_types() const {
+  return _config->num_mem_object_types();
+}
+
+
+G1CardSetMemoryManager::~G1CardSetMemoryManager() {
+  for (uint i = 0; i < num_mem_object_types(); i++) {
+    _allocators[i].~G1CardSetAllocator();
+  }
+  FREE_C_HEAP_ARRAY(G1CardSetAllocator<G1CardSetContainer>, _allocators);
+}
+
+void G1CardSetMemoryManager::free(uint type, void* value) {
+  assert(type < num_mem_object_types(), "must be");
+  _allocators[type].free((G1CardSetContainer*)value);
+}
+
+void G1CardSetMemoryManager::flush() {
+  for (uint i = 0; i < num_mem_object_types(); i++) {
+    _allocators[i].drop_all();
+  }
+}
+
+void G1CardSetMemoryManager::print(outputStream* os) {
+  os->print_cr("MM " PTR_FORMAT " size %zu", p2i(this), sizeof(*this));
+  for (uint i = 0; i < num_mem_object_types(); i++) {
+    _allocators[i].print(os);
+  }
+}
+
+size_t G1CardSetMemoryManager::mem_size() const {
+  size_t result = 0;
+  for (uint i = 0; i < num_mem_object_types(); i++) {
+    result += _allocators[i].mem_size();
+  }
+  return sizeof(*this) -
+    (sizeof(G1CardSetAllocator<G1CardSetContainer>) * num_mem_object_types()) +
+    result;
+}
+
+size_t G1CardSetMemoryManager::wasted_mem_size() const {
+  size_t result = 0;
+  for (uint i = 0; i < num_mem_object_types(); i++) {
+    result += _allocators[i].wasted_mem_size();
+  }
+  return result;
+}
+
+G1CardSetMemoryStats G1CardSetMemoryManager::memory_stats() const {
+  G1CardSetMemoryStats result;
+  for (uint i = 0; i < num_mem_object_types(); i++) {
+    result._num_mem_sizes[i] += _allocators[i].mem_size();
+    result._num_buffers[i] += _allocators[i].num_buffers();
+  }
+  return result;
+}

src/hotspot/share/gc/g1/g1CardSetMemory.hpp

@@ -0,0 +1,384 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1CARDSETMEMORY_HPP
+#define SHARE_GC_G1_G1CARDSETMEMORY_HPP
+
+#include "gc/g1/g1CardSet.hpp"
+#include "gc/g1/g1CardSetContainers.hpp"
+#include "memory/allocation.hpp"
+#include "utilities/growableArray.hpp"
+#include "utilities/lockFreeStack.hpp"
+
+class G1CardSetConfiguration;
+class outputStream;
+
+// Collects G1CardSetAllocator options/heuristics. Called by G1CardSetAllocator
+// to determine the next size of the allocated G1CardSetBuffer.
+class G1CardSetAllocOptions {
+  uint _elem_size;
+  uint _initial_num_elems;
+  // Defines a limit to the number of elements in the buffer
+  uint _max_num_elems;
+
+  uint exponential_expand(uint prev_num_elems) {
+    return clamp(prev_num_elems * 2, _initial_num_elems, _max_num_elems);
+  }
+
+public:
+  static const uint BufferAlignment = 8;
+  static const uint MinimumBufferSize = 8;
+  static const uint MaximumBufferSize =  UINT_MAX / 2;
+
+  G1CardSetAllocOptions(uint elem_size, uint initial_num_elems = MinimumBufferSize, uint max_num_elems = MaximumBufferSize) :
+    _elem_size(align_up(elem_size, BufferAlignment)),
+    _initial_num_elems(initial_num_elems),
+    _max_num_elems(max_num_elems) {
+  }
+
+  uint next_num_elems(uint prev_num_elems) {
+    return exponential_expand(prev_num_elems);
+  }
+
+  uint elem_size () const {return _elem_size;}
+};
+
+// A single buffer/arena containing _num_elems blocks of memory of _elem_size.
+// G1CardSetBuffers can be linked together using a singly linked list.
+class G1CardSetBuffer : public CHeapObj<mtGCCardSet> {
+  uint _elem_size;
+  uint _num_elems;
+
+  G1CardSetBuffer* volatile _next;
+
+  char* _buffer;  // Actual data.
+
+  // Index into the next free block to allocate into. Full if equal (or larger)
+  // to _num_elems (can be larger because we atomically increment this value and
+  // check only afterwards if the allocation has been successful).
+  uint volatile _next_allocate;
+
+public:
+  G1CardSetBuffer(uint elem_size, uint num_elems, G1CardSetBuffer* next);
+  ~G1CardSetBuffer();
+
+  G1CardSetBuffer* volatile* next_addr() { return &_next; }
+
+  void* get_new_buffer_elem();
+
+  uint num_elems() const { return _num_elems; }
+
+  G1CardSetBuffer* next() const { return _next; }
+
+  void set_next(G1CardSetBuffer* next) {
+    assert(next != this, " loop condition");
+    _next = next;
+  }
+
+  void reset(G1CardSetBuffer* next) {
+    _next_allocate = 0;
+    assert(next != this, " loop condition");
+    set_next(next);
+    memset((void*)_buffer, 0, (size_t)_num_elems * _elem_size);
+  }
+
+  uint elem_size() const { return _elem_size; }
+
+  size_t mem_size() const { return sizeof(*this) + (size_t)_num_elems * _elem_size; }
+
+  bool is_full() const { return _next_allocate >= _num_elems; }
+};
+
+// Set of (free) G1CardSetBuffers. The assumed usage is that allocation
+// to it and removal of elements is strictly separate, but every action may be
+// performed by multiple threads at the same time.
+// Counts and memory usage are current on a best-effort basis if accessed concurrently.
+class G1CardSetBufferList {
+  static G1CardSetBuffer* volatile* next_ptr(G1CardSetBuffer& node) {
+    return node.next_addr();
+  }
+  typedef LockFreeStack<G1CardSetBuffer, &next_ptr> NodeStack;
+
+  NodeStack _list;
+
+  volatile size_t _num_buffers;
+  volatile size_t _mem_size;
+
+public:
+  G1CardSetBufferList() : _list(), _num_buffers(0), _mem_size(0) { }
+  ~G1CardSetBufferList() { free_all(); }
+
+  void bulk_add(G1CardSetBuffer& first, G1CardSetBuffer& last, size_t num, size_t mem_size);
+  void add(G1CardSetBuffer& elem) { _list.prepend(elem); }
+
+  G1CardSetBuffer* get();
+  G1CardSetBuffer* get_all(size_t& num_buffers, size_t& mem_size);
+
+  // Give back all memory to the OS.
+  void free_all();
+
+  void print_on(outputStream* out, const char* prefix = "");
+
+  size_t num_buffers() const { return Atomic::load(&_num_buffers); }
+  size_t mem_size() const { return Atomic::load(&_mem_size); }
+};
+
+// Arena-like allocator for (card set) heap memory objects (Elem elements).
+//
+// Actual allocation from the C heap occurs on G1CardSetBuffer basis, i.e. sets
+// of elements. The assumed allocation pattern for these G1CardSetBuffer elements
+// is assumed to be strictly two-phased:
+//
+// - in the first phase, G1CardSetBuffers are allocated from the C heap (or a free
+// list given at initialization time). This allocation may occur in parallel. This
+// typically corresponds to a single mutator phase, but may extend over multiple.
+//
+// - in the second phase, G1CardSetBuffers are given back in bulk to the free list.
+// This is typically done during a GC pause.
+//
+// Some third party is responsible for giving back memory from the free list to
+// the operating system.
+//
+// Allocation and deallocation in the first phase on G1CardSetContainer basis
+// may occur by multiple threads at once.
+//
+// Allocation occurs from an internal free list of G1CardSetContainers first,
+// only then trying to bump-allocate from the current G1CardSetBuffer. If there is
+// none, this class allocates a new G1CardSetBuffer (allocated from the C heap,
+// asking the G1CardSetAllocOptions instance about sizes etc) and uses that one.
+//
+// The G1CardSetContainerOnHeaps free list is a linked list of G1CardSetContainers
+// within all G1CardSetBuffer instances allocated so far. It uses a separate
+// pending list and global synchronization to avoid the ABA problem when the
+// user frees a memory object.
+//
+// The class also manages a few counters for statistics using atomic operations.
+// Their values are only consistent within each other with extra global
+// synchronization.
+//
+// Since it is expected that every CardSet (and in extension each region) has its
+// own set of allocators, there is intentionally no padding between them to save
+// memory.
+template <class Elem>
+class G1CardSetAllocator {
+  // G1CardSetBuffer management.
+
+  // G1CardSetAllocOptions provides parameters for allocation buffer
+  // sizing and expansion.
+  G1CardSetAllocOptions _alloc_options;
+
+  G1CardSetBuffer* volatile _first;       // The (start of the) list of all buffers.
+  G1CardSetBuffer* _last;                 // The last element of the list of all buffers.
+  volatile uint _num_buffers;             // Number of assigned buffers to this allocator.
+  volatile size_t _mem_size;              // Memory used by all buffers.
+
+  G1CardSetBufferList* _free_buffer_list; // The global free buffer list to
+                                          // preferentially get new buffers from.
+
+  // G1CardSetContainer node management within the G1CardSetBuffers allocated
+  // by this allocator.
+
+  static G1CardSetContainer* volatile* next_ptr(G1CardSetContainer& node);
+  typedef LockFreeStack<G1CardSetContainer, &next_ptr> NodeStack;
+
+  volatile bool _transfer_lock;
+  NodeStack _free_nodes_list;
+  NodeStack _pending_nodes_list;
+
+  volatile uint _num_pending_nodes;   // Number of nodes in the pending list.
+  volatile uint _num_free_nodes;      // Number of nodes in the free list.
+
+  volatile uint _num_allocated_nodes; // Number of total nodes allocated and in use.
+  volatile uint _num_available_nodes; // Number of nodes available in all buffers (allocated + free + pending + not yet used).
+
+  // Try to transfer nodes from _pending_nodes_list to _free_nodes_list, with a
+  // synchronization delay for any in-progress pops from the _free_nodes_list
+  // to solve ABA here.
+  bool try_transfer_pending();
+
+  uint num_free_elems() const;
+
+  G1CardSetBuffer* create_new_buffer(G1CardSetBuffer* const prev);
+
+  uint elem_size() const { return _alloc_options.elem_size(); }
+
+public:
+  G1CardSetAllocator(const char* name,
+                     const G1CardSetAllocOptions& buffer_options,
+                     G1CardSetBufferList* free_buffer_list);
+  ~G1CardSetAllocator() {
+    drop_all();
+  }
+
+  Elem* allocate();
+  void free(Elem* elem);
+
+  // Deallocate all buffers to the free buffer list and reset this allocator. Must
+  // be called in a globally synchronized area.
+  void drop_all();
+
+  uint num_buffers() const;
+
+  size_t mem_size() const {
+    return sizeof(*this) +
+      num_buffers() * sizeof(G1CardSetBuffer) + (size_t)_num_available_nodes * elem_size();
+  }
+
+  size_t wasted_mem_size() const {
+    return ((size_t)_num_available_nodes - (_num_allocated_nodes - _num_pending_nodes)) * elem_size();
+  }
+
+  void print(outputStream* os);
+};
+
+// Statistics for a fixed set of buffer lists. Contains the number of buffers and memory
+// used for each. Note that statistics are typically not taken atomically so there
+// can be inconsistencies. The user must be prepared for them.
+class G1CardSetMemoryStats {
+public:
+
+  size_t _num_mem_sizes[G1CardSetConfiguration::num_mem_object_types()];
+  size_t _num_buffers[G1CardSetConfiguration::num_mem_object_types()];
+
+  // Returns all-zero statistics.
+  G1CardSetMemoryStats();
+  // For every element in the set (indicated by i), call fn to provide the
+  // memory size and number of buffers for that i'th buffer list.
+  G1CardSetMemoryStats(void (*fn)(const void* context, uint i, size_t& mem_size, size_t& num_buffers), const void* context);
+
+  void add(G1CardSetMemoryStats const other) {
+    STATIC_ASSERT(ARRAY_SIZE(_num_buffers) == ARRAY_SIZE(_num_mem_sizes));
+    for (uint i = 0; i < ARRAY_SIZE(_num_mem_sizes); i++) {
+      _num_mem_sizes[i] += other._num_mem_sizes[i];
+      _num_buffers[i] += other._num_buffers[i];
+    }
+  }
+
+  void clear();
+
+  uint num_pools() const { return G1CardSetConfiguration::num_mem_object_types(); }
+};
+
+// A set of free lists holding memory buffers for use by G1CardSetAllocators.
+class G1CardSetFreePool {
+  // The global free pool.
+  static G1CardSetFreePool _freelist_pool;
+
+  uint _num_free_lists;
+  G1CardSetBufferList* _free_lists;
+
+public:
+  static G1CardSetFreePool* free_list_pool() { return &_freelist_pool; }
+  static G1CardSetMemoryStats free_list_sizes() { return _freelist_pool.memory_sizes(); }
+
+  class G1ReturnMemoryProcessor;
+  typedef GrowableArrayCHeap<G1ReturnMemoryProcessor*, mtGC> G1ReturnMemoryProcessorSet;
+
+  static void update_unlink_processors(G1ReturnMemoryProcessorSet* unlink_processors);
+
+  explicit G1CardSetFreePool(uint num_free_lists);
+  ~G1CardSetFreePool();
+
+  G1CardSetBufferList* free_list(uint i) {
+    assert(i < _num_free_lists, "must be");
+    return &_free_lists[i];
+  }
+
+  uint num_free_lists() const { return _num_free_lists; }
+
+  // Return sizes for free list i in this free list pool.
+  void get_size(uint i, size_t& mem_size, size_t& num_buffers) const;
+
+  G1CardSetMemoryStats memory_sizes() const;
+  size_t mem_size() const;
+
+  void print_on(outputStream* out);
+};
+
+// Data structure containing current in-progress state for returning memory to the
+// operating system for a single G1CardSetBufferList.
+class G1CardSetFreePool::G1ReturnMemoryProcessor : public CHeapObj<mtGC> {
+  G1CardSetBufferList* _source;
+  size_t _return_to_vm_size;
+
+  G1CardSetBuffer* _first;
+  size_t _unlinked_bytes;
+  size_t _num_unlinked;
+
+public:
+  explicit G1ReturnMemoryProcessor(size_t return_to_vm) :
+    _source(nullptr), _return_to_vm_size(return_to_vm), _first(nullptr), _unlinked_bytes(0), _num_unlinked(0) {
+  }
+
+  // Updates the instance members about the given card set buffer list for the purpose
+  // of giving back memory. Only necessary members are updated, e.g. if there is
+  // nothing to return to the VM, do not set the source list.
+  void visit_free_list(G1CardSetBufferList* source);
+
+  bool finished_return_to_vm() const { return _return_to_vm_size == 0; }
+  bool finished_return_to_os() const { return _first == nullptr; }
+
+  // Returns memory to the VM until the given deadline expires. Returns true if
+  // there is no more work. Guarantees forward progress, i.e. at least one buffer
+  // has been processed after returning.
+  // return_to_vm() re-adds buffers to the respective free list.
+  bool return_to_vm(jlong deadline);
+  // Returns memory to the VM until the given deadline expires. Returns true if
+  // there is no more work. Guarantees forward progress, i.e. at least one buffer
+  // has been processed after returning.
+  // return_to_os() gives back buffers to the OS.
+  bool return_to_os(jlong deadline);
+};
+
+class G1CardSetMemoryManager : public CHeapObj<mtGCCardSet> {
+  G1CardSetConfiguration* _config;
+
+  G1CardSetAllocator<G1CardSetContainer>* _allocators;
+
+  uint num_mem_object_types() const;
+public:
+  G1CardSetMemoryManager(G1CardSetConfiguration* config,
+                         G1CardSetFreePool* free_list_pool);
+
+  virtual ~G1CardSetMemoryManager();
+
+  // Allocate and free a memory object of given type.
+  inline uint8_t* allocate(uint type);
+  void free(uint type, void* value);
+
+  // Allocate and free a hash table node.
+  inline uint8_t* allocate_node();
+  inline void free_node(void* value);
+
+  void flush();
+
+  void print(outputStream* os);
+
+  size_t mem_size() const;
+  size_t wasted_mem_size() const;
+
+  G1CardSetMemoryStats memory_stats() const;
+};
+
+#endif // SHARE_GC_G1_G1CARDSETMEMORY_HPP

src/hotspot/share/gc/g1/g1CardSetMemory.inline.hpp

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1CARDSETMEMORY_INLINE_HPP
+#define SHARE_GC_G1_G1CARDSETMEMORY_INLINE_HPP
+
+#include "gc/g1/g1CardSetMemory.hpp"
+#include "gc/g1/g1CardSetContainers.hpp"
+#include "utilities/ostream.hpp"
+
+#include "gc/g1/g1CardSetContainers.inline.hpp"
+#include "utilities/globalCounter.inline.hpp"
+
+template <class Elem>
+G1CardSetContainer* volatile* G1CardSetAllocator<Elem>::next_ptr(G1CardSetContainer& node) {
+  return node.next_addr();
+}
+
+template <class Elem>
+G1CardSetBuffer* G1CardSetAllocator<Elem>::create_new_buffer(G1CardSetBuffer* const prev) {
+
+  // Take an existing buffer if available.
+  G1CardSetBuffer* next = _free_buffer_list->get();
+  if (next == nullptr) {
+    uint prev_num_elems = (prev != nullptr) ? prev->num_elems() : 0;
+    uint num_elems = _alloc_options.next_num_elems(prev_num_elems);
+    next = new G1CardSetBuffer(elem_size(), num_elems, prev);
+  } else {
+    assert(elem_size() == next->elem_size() , "Mismatch %d != %d Elem %zu", elem_size(), next->elem_size(), sizeof(Elem));
+    next->reset(prev);
+  }
+
+  // Install it as current allocation buffer.
+  G1CardSetBuffer* old = Atomic::cmpxchg(&_first, prev, next);
+  if (old != prev) {
+    // Somebody else installed the buffer, use that one.
+    delete next;
+    return old;
+  } else {
+    // Did we install the first element in the list? If so, this is also the last.
+    if (prev == nullptr) {
+      _last = next;
+    }
+    // Successfully installed the buffer into the list.
+    Atomic::inc(&_num_buffers, memory_order_relaxed);
+    Atomic::add(&_mem_size, next->mem_size(), memory_order_relaxed);
+    Atomic::add(&_num_available_nodes, next->num_elems(), memory_order_relaxed);
+    return next;
+  }
+}
+
+template <class Elem>
+Elem* G1CardSetAllocator<Elem>::allocate() {
+  assert(elem_size() > 0, "instance size not set.");
+
+  if (num_free_elems() > 0) {
+    // Pop under critical section to deal with ABA problem
+    // Other solutions to the same problem are more complicated (ref counting, HP)
+    GlobalCounter::CriticalSection cs(Thread::current());
+
+    G1CardSetContainer* node = _free_nodes_list.pop();
+    if (node != nullptr) {
+      Elem* elem = reinterpret_cast<Elem*>(reinterpret_cast<char*>(node));
+      Atomic::sub(&_num_free_nodes, 1u);
+      guarantee(is_aligned(elem, 8), "result " PTR_FORMAT " not aligned", p2i(elem));
+      return elem;
+    }
+  }
+
+  G1CardSetBuffer* cur = Atomic::load_acquire(&_first);
+  if (cur == nullptr) {
+    cur = create_new_buffer(cur);
+  }
+
+  while (true) {
+    Elem* elem = (Elem*)cur->get_new_buffer_elem();
+    if (elem != nullptr) {
+      Atomic::inc(&_num_allocated_nodes, memory_order_relaxed);
+      guarantee(is_aligned(elem, 8), "result " PTR_FORMAT " not aligned", p2i(elem));
+      return elem;
+    }
+    // The buffer is full. Next round.
+    assert(cur->is_full(), "must be");
+    cur = create_new_buffer(cur);
+  }
+}
+
+inline uint8_t* G1CardSetMemoryManager::allocate(uint type) {
+  assert(type < num_mem_object_types(), "must be");
+  return (uint8_t*)_allocators[type].allocate();
+}
+
+inline uint8_t* G1CardSetMemoryManager::allocate_node() {
+  return allocate(0);
+}
+
+inline void G1CardSetMemoryManager::free_node(void* value) {
+  free(0, value);
+}
+
+template <class Elem>
+inline uint G1CardSetAllocator<Elem>::num_buffers() const {
+  return Atomic::load(&_num_buffers);
+}
+
+template <class Elem>
+inline uint G1CardSetAllocator<Elem>::num_free_elems() const {
+  return Atomic::load(&_num_free_nodes);
+}
+
+#endif // SHARE_GC_G1_G1CARDSETMEMORY_INLINE_HPP

src/hotspot/share/gc/g1/g1CardTable.hpp

@@ -102,7 +102,7 @@ public:
   // already scanned cards. Assumes that most cards in that area are Clean.
   // Returns the number of dirtied cards that were not yet dirty. This result may
   // be inaccurate as it does not perform the dirtying atomically.
-  inline size_t mark_region_dirty(size_t start_card_index, size_t num_cards);
+  inline size_t mark_range_dirty(size_t start_card_index, size_t num_cards);
 
   // Change the given range of dirty cards to "which". All of these cards must be Dirty.
   inline void change_dirty_cards_to(size_t start_card_index, size_t num_cards, CardValue which);

src/hotspot/share/gc/g1/g1CardTable.inline.hpp

@@ -43,7 +43,7 @@ inline bool G1CardTable::mark_clean_as_dirty(CardValue* card) {
   return false;
 }
 
-inline size_t G1CardTable::mark_region_dirty(size_t start_card_index, size_t num_cards) {
+inline size_t G1CardTable::mark_range_dirty(size_t start_card_index, size_t num_cards) {
   assert(is_aligned(start_card_index, sizeof(size_t)), "Start card index must be aligned.");
   assert(is_aligned(num_cards, sizeof(size_t)), "Number of cards to change must be evenly divisible.");
 

src/hotspot/share/gc/g1/g1CodeBlobClosure.cpp

@@ -28,7 +28,7 @@
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1ConcurrentMark.inline.hpp"
 #include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/compressedOops.inline.hpp"
 #include "oops/oop.inline.hpp"

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -34,6 +34,7 @@
 #include "gc/g1/g1BarrierSet.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1CollectionSet.hpp"
+#include "gc/g1/g1CollectionSetCandidates.hpp"
 #include "gc/g1/g1CollectorState.hpp"
 #include "gc/g1/g1ConcurrentRefine.hpp"
 #include "gc/g1/g1ConcurrentRefineThread.hpp"
@@ -68,7 +69,7 @@
 #include "gc/g1/g1VMOperations.hpp"
 #include "gc/g1/g1YoungGCPostEvacuateTasks.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/g1/heapRegionSet.inline.hpp"
 #include "gc/shared/concurrentGCBreakpoints.hpp"
 #include "gc/shared/gcBehaviours.hpp"
@@ -147,7 +148,7 @@ void G1CollectedHeap::run_batch_task(G1BatchedGangTask* cl) {
 
 HeapRegion* G1CollectedHeap::new_heap_region(uint hrs_index,
                                              MemRegion mr) {
-  return new HeapRegion(hrs_index, bot(), mr);
+  return new HeapRegion(hrs_index, bot(), mr, &_card_set_config);
 }
 
 // Private methods.
@@ -1436,6 +1437,7 @@ G1CollectedHeap::G1CollectedHeap() :
   CollectedHeap(),
   _service_thread(NULL),
   _periodic_gc_task(NULL),
+  _free_card_set_memory_task(NULL),
   _workers(NULL),
   _card_table(NULL),
   _collection_pause_end(Ticks::now()),
@@ -1472,6 +1474,7 @@ G1CollectedHeap::G1CollectedHeap() :
   _collection_set(this, _policy),
   _hot_card_cache(NULL),
   _rem_set(NULL),
+  _card_set_config(),
   _cm(NULL),
   _cm_thread(NULL),
   _cr(NULL),
@@ -1741,6 +1744,9 @@ jint G1CollectedHeap::initialize() {
   _periodic_gc_task = new G1PeriodicGCTask("Periodic GC Task");
   _service_thread->register_task(_periodic_gc_task);
 
+  _free_card_set_memory_task = new G1CardSetFreeMemoryTask("Card Set Free Memory Task");
+  _service_thread->register_task(_free_card_set_memory_task);
+
   {
     G1DirtyCardQueueSet& dcqs = G1BarrierSet::dirty_card_queue_set();
     dcqs.set_process_cards_threshold(concurrent_refine()->yellow_zone());
@@ -2621,6 +2627,9 @@ void G1CollectedHeap::gc_epilogue(bool full) {
   _numa->print_statistics();
 
   _collection_pause_end = Ticks::now();
+
+  _free_card_set_memory_task->notify_new_stats(&_young_gen_card_set_stats,
+                                               &_collection_set_candidates_card_set_stats);
 }
 
 uint G1CollectedHeap::uncommit_regions(uint region_limit) {
@@ -2958,7 +2967,6 @@ void G1CollectedHeap::do_collection_pause_at_safepoint_helper(double target_paus
                                                             Threads::number_of_non_daemon_threads());
     active_workers = workers()->update_active_workers(active_workers);
     log_info(gc,task)("Using %u workers of %u for evacuation", active_workers, workers()->total_workers());
-
     G1MonitoringScope ms(g1mm(),
                          false /* full_gc */,
                          collector_state()->in_mixed_phase() /* all_memory_pools_affected */);
@@ -2967,12 +2975,10 @@ void G1CollectedHeap::do_collection_pause_at_safepoint_helper(double target_paus
 
     {
       IsGCActiveMark x;
-
       gc_prologue(false);
 
       G1HeapVerifier::G1VerifyType verify_type = young_collection_verify_type();
       verify_before_young_collection(verify_type);
-
       {
         // The elapsed time induced by the start time below deliberately elides
         // the possible verification above.
@@ -3319,6 +3325,15 @@ bool G1CollectedHeap::should_do_eager_reclaim() const {
           (has_humongous_reclaim_candidates() || do_humongous_object_logging()));
 }
 
+bool G1CollectedHeap::should_sample_collection_set_candidates() const {
+  G1CollectionSetCandidates* candidates = G1CollectedHeap::heap()->collection_set()->candidates();
+  return candidates != NULL && candidates->num_remaining() > 0;
+}
+
+void G1CollectedHeap::set_collection_set_candidates_stats(G1CardSetMemoryStats& stats) {
+  _collection_set_candidates_card_set_stats = stats;
+}
+
 class G1PrepareEvacuationTask : public AbstractGangTask {
   class G1PrepareRegionsClosure : public HeapRegionClosure {
     G1CollectedHeap* _g1h;
@@ -3326,6 +3341,17 @@ class G1PrepareEvacuationTask : public AbstractGangTask {
     uint _worker_humongous_total;
     uint _worker_humongous_candidates;
 
+    G1CardSetMemoryStats _card_set_stats;
+
+    void sample_card_set_size(HeapRegion* hr) {
+      // Sample card set sizes for young gen and humongous before GC: this makes
+      // the policy to give back memory to the OS keep the most recent amount of
+      // memory for these regions.
+      if (hr->is_young() || hr->is_starts_humongous()) {
+        _card_set_stats.add(hr->rem_set()->card_set_memory_stats());
+      }
+    }
+
     bool humongous_region_is_candidate(HeapRegion* region) const {
       assert(region->is_starts_humongous(), "Must start a humongous object");
 
@@ -3398,6 +3424,8 @@ class G1PrepareEvacuationTask : public AbstractGangTask {
       // First prepare the region for scanning
       _g1h->rem_set()->prepare_region_for_scan(hr);
 
+      sample_card_set_size(hr);
+
       // Now check if region is a humongous candidate
       if (!hr->is_starts_humongous()) {
         _g1h->register_region_with_region_attr(hr);
@@ -3428,12 +3456,19 @@ class G1PrepareEvacuationTask : public AbstractGangTask {
 
       return false;
     }
+
+    G1CardSetMemoryStats card_set_stats() const {
+      return _card_set_stats;
+    }
   };
 
   G1CollectedHeap* _g1h;
   HeapRegionClaimer _claimer;
   volatile uint _humongous_total;
   volatile uint _humongous_candidates;
+
+  G1CardSetMemoryStats _all_card_set_stats;
+
 public:
   G1PrepareEvacuationTask(G1CollectedHeap* g1h) :
     AbstractGangTask("Prepare Evacuation"),
@@ -3445,6 +3480,9 @@ public:
   void work(uint worker_id) {
     G1PrepareRegionsClosure cl(_g1h, this);
     _g1h->heap_region_par_iterate_from_worker_offset(&cl, &_claimer, worker_id);
+
+    MutexLocker x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
+    _all_card_set_stats.add(cl.card_set_stats());
   }
 
   void add_humongous_candidates(uint candidates) {
@@ -3462,6 +3500,10 @@ public:
   uint humongous_total() {
     return _humongous_total;
   }
+
+  G1CardSetMemoryStats all_card_set_stats() const {
+    return _all_card_set_stats;
+  }
 };
 
 void G1CollectedHeap::pre_evacuate_collection_set(G1EvacuationInfo& evacuation_info, G1ParScanThreadStateSet* per_thread_states) {
@@ -3489,6 +3531,8 @@ void G1CollectedHeap::pre_evacuate_collection_set(G1EvacuationInfo& evacuation_i
     G1PrepareEvacuationTask g1_prep_task(this);
     Tickspan task_time = run_task_timed(&g1_prep_task);
 
+    _young_gen_card_set_stats = g1_prep_task.all_card_set_stats();
+
     phase_times()->record_register_regions(task_time.seconds() * 1000.0);
     _num_humongous_objects = g1_prep_task.humongous_total();
     _num_humongous_reclaim_candidates = g1_prep_task.humongous_candidates();
@@ -3763,6 +3807,7 @@ void G1CollectedHeap::post_evacuate_collection_set(G1EvacuationInfo& evacuation_
   evacuation_info.set_bytes_used(_bytes_used_during_gc);
 
   policy()->print_age_table();
+  rem_set()->print_coarsen_stats();
 }
 
 void G1CollectedHeap::record_obj_copy_mem_stats() {

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -27,6 +27,8 @@
 
 #include "gc/g1/g1BarrierSet.hpp"
 #include "gc/g1/g1BiasedArray.hpp"
+#include "gc/g1/g1CardSet.hpp"
+#include "gc/g1/g1CardSetFreeMemoryTask.hpp"
 #include "gc/g1/g1CardTable.hpp"
 #include "gc/g1/g1CollectionSet.hpp"
 #include "gc/g1/g1CollectorState.hpp"
@@ -66,6 +68,7 @@
 // Forward declarations
 class HeapRegion;
 class GenerationSpec;
+class G1CardSetFreeMemoryTask;
 class G1ParScanThreadState;
 class G1ParScanThreadStateSet;
 class G1ParScanThreadState;
@@ -158,6 +161,7 @@ class G1CollectedHeap : public CollectedHeap {
 private:
   G1ServiceThread* _service_thread;
   G1ServiceTask* _periodic_gc_task;
+  G1CardSetFreeMemoryTask* _free_card_set_memory_task;
 
   WorkGang* _workers;
   G1CardTable* _card_table;
@@ -173,6 +177,11 @@ private:
   HeapRegionSet _archive_set;
   HeapRegionSet _humongous_set;
 
+  // Young gen memory statistics before GC.
+  G1CardSetMemoryStats _young_gen_card_set_stats;
+  // Collection set candidates memory statistics after GC.
+  G1CardSetMemoryStats _collection_set_candidates_card_set_stats;
+
   void rebuild_free_region_list();
   // Start a new incremental collection set for the next pause.
   void start_new_collection_set();
@@ -267,6 +276,9 @@ public:
 
   bool should_do_eager_reclaim() const;
 
+  bool should_sample_collection_set_candidates() const;
+  void set_collection_set_candidates_stats(G1CardSetMemoryStats& stats);
+
 private:
 
   G1HRPrinter _hr_printer;
@@ -840,6 +852,8 @@ public:
 
   // The g1 remembered set of the heap.
   G1RemSet* _rem_set;
+  // Global card set configuration
+  G1CardSetConfiguration _card_set_config;
 
   void post_evacuate_cleanup_1(G1ParScanThreadStateSet* per_thread_states,
                                G1RedirtyCardsQueueSet* rdcqs);

src/hotspot/share/gc/g1/g1CollectionSet.cpp

@@ -31,7 +31,7 @@
 #include "gc/g1/g1ParScanThreadState.hpp"
 #include "gc/g1/g1Policy.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/g1/heapRegionSet.hpp"
 #include "logging/logStream.hpp"
 #include "runtime/orderAccess.hpp"

src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp

@@ -26,7 +26,7 @@
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1CollectionSetCandidates.hpp"
 #include "gc/g1/g1CollectionSetChooser.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/shared/space.inline.hpp"
 #include "runtime/atomic.hpp"
 #include "utilities/quickSort.hpp"

src/hotspot/share/gc/g1/g1ConcurrentMark.cpp

@@ -27,6 +27,7 @@
 #include "classfile/systemDictionary.hpp"
 #include "code/codeCache.hpp"
 #include "gc/g1/g1BarrierSet.hpp"
+#include "gc/g1/g1CardSetMemory.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1CollectorState.hpp"
 #include "gc/g1/g1ConcurrentMark.inline.hpp"
@@ -39,7 +40,7 @@
 #include "gc/g1/g1ThreadLocalData.hpp"
 #include "gc/g1/g1Trace.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/g1/heapRegionSet.inline.hpp"
 #include "gc/shared/gcId.hpp"
 #include "gc/shared/gcTimer.hpp"
@@ -2947,7 +2948,7 @@ G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() {
   }
 
   // add static memory usages to remembered set sizes
-  _total_remset_bytes += HeapRegionRemSet::fl_mem_size() + HeapRegionRemSet::static_mem_size();
+  _total_remset_bytes += G1CardSetFreePool::free_list_pool()->mem_size() + HeapRegionRemSet::static_mem_size();
   // Print the footer of the output.
   log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
   log_trace(gc, liveness)(G1PPRL_LINE_PREFIX

src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp

@@ -34,7 +34,7 @@
 #include "gc/g1/g1Policy.hpp"
 #include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
 #include "gc/g1/g1RemSetTrackingPolicy.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/g1/heapRegion.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "gc/shared/taskqueue.inline.hpp"

src/hotspot/share/gc/g1/g1DirtyCardQueue.cpp

@@ -34,7 +34,7 @@
 #include "gc/g1/g1RedirtyCardsQueue.hpp"
 #include "gc/g1/g1RemSet.hpp"
 #include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "memory/iterator.hpp"
 #include "runtime/atomic.hpp"

src/hotspot/share/gc/g1/g1EvacFailure.cpp

@@ -31,7 +31,7 @@
 #include "gc/g1/g1OopClosures.inline.hpp"
 #include "gc/g1/g1RedirtyCardsQueue.hpp"
 #include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/shared/preservedMarks.inline.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/compressedOops.inline.hpp"

src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp

@@ -31,7 +31,7 @@
 #include "gc/g1/g1FullCollector.inline.hpp"
 #include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
 #include "gc/g1/g1FullGCMarker.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "memory/iterator.inline.hpp"
 #include "memory/universe.hpp"
 #include "oops/access.inline.hpp"

src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp

@@ -41,6 +41,8 @@
 #include "utilities/enumIterator.hpp"
 #include "utilities/macros.hpp"
 
+constexpr const char* G1GCPhaseTimes::GCMergeRSWorkItemsStrings[];
+
 G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) :
   _max_gc_threads(max_gc_threads),
   _gc_start_counter(0),
@@ -71,16 +73,14 @@ G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) :
   _gc_par_phases[MergeER] = new WorkerDataArray<double>("MergeER", "Eager Reclaim (ms):", max_gc_threads);
 
   _gc_par_phases[MergeRS] = new WorkerDataArray<double>("MergeRS", "Remembered Sets (ms):", max_gc_threads);
-  _gc_par_phases[MergeRS]->create_thread_work_items("Merged Sparse:", MergeRSMergedSparse);
-  _gc_par_phases[MergeRS]->create_thread_work_items("Merged Fine:", MergeRSMergedFine);
-  _gc_par_phases[MergeRS]->create_thread_work_items("Merged Coarse:", MergeRSMergedCoarse);
-  _gc_par_phases[MergeRS]->create_thread_work_items("Dirty Cards:", MergeRSDirtyCards);
+  for (uint i = 0; i < MergeRSContainersSentinel; i++) {
+    _gc_par_phases[MergeRS]->create_thread_work_items(GCMergeRSWorkItemsStrings[i], i);
+  }
 
   _gc_par_phases[OptMergeRS] = new WorkerDataArray<double>("OptMergeRS", "Optional Remembered Sets (ms):", max_gc_threads);
-  _gc_par_phases[OptMergeRS]->create_thread_work_items("Merged Sparse:", MergeRSMergedSparse);
-  _gc_par_phases[OptMergeRS]->create_thread_work_items("Merged Fine:", MergeRSMergedFine);
-  _gc_par_phases[OptMergeRS]->create_thread_work_items("Merged Coarse:", MergeRSMergedCoarse);
-  _gc_par_phases[OptMergeRS]->create_thread_work_items("Dirty Cards:", MergeRSDirtyCards);
+  for (uint i = 0; i < MergeRSContainersSentinel; i++) {
+    _gc_par_phases[OptMergeRS]->create_thread_work_items(GCMergeRSWorkItemsStrings[i], i);
+  }
 
   _gc_par_phases[MergeLB] = new WorkerDataArray<double>("MergeLB", "Log Buffers (ms):", max_gc_threads);
   if (G1HotCardCache::default_use_cache()) {
@@ -134,6 +134,8 @@ G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) :
   _gc_par_phases[EagerlyReclaimHumongousObjects]->create_thread_work_items("Humongous Candidates", EagerlyReclaimNumCandidates);
   _gc_par_phases[EagerlyReclaimHumongousObjects]->create_thread_work_items("Humongous Reclaimed", EagerlyReclaimNumReclaimed);
 
+  _gc_par_phases[SampleCollectionSetCandidates] = new WorkerDataArray<double>("SampleCandidates", "Sample CSet Candidates (ms):", max_gc_threads);
+
   _gc_par_phases[Termination]->create_thread_work_items("Termination Attempts:");
 
   _gc_par_phases[OptTermination]->create_thread_work_items("Optional Termination Attempts:");
@@ -170,6 +172,7 @@ void G1GCPhaseTimes::reset() {
   _recorded_clear_claimed_marks_time_ms = 0.0;
   _recorded_young_cset_choice_time_ms = 0.0;
   _recorded_non_young_cset_choice_time_ms = 0.0;
+  _recorded_sample_collection_set_candidates_time_ms = 0.0;
   _recorded_preserve_cm_referents_time_ms = 0.0;
   _recorded_start_new_cset_time_ms = 0.0;
   _recorded_serial_free_cset_time_ms = 0.0;
@@ -449,6 +452,7 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set() const {
                         _recorded_preserve_cm_referents_time_ms +
                         _cur_ref_proc_time_ms +
                         (_weak_phase_times.total_time_sec() * MILLIUNITS) +
+                        _recorded_sample_collection_set_candidates_time_ms +
                         _cur_post_evacuate_cleanup_1_time_ms +
                         _cur_post_evacuate_cleanup_2_time_ms +
                         _recorded_total_rebuild_freelist_time_ms +
@@ -472,6 +476,8 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set() const {
     debug_phase(_gc_par_phases[RemoveSelfForwardingPtr], 1);
   }
 
+  debug_time("Sample Collection Set Candidates", _recorded_sample_collection_set_candidates_time_ms);
+  trace_phase(_gc_par_phases[RedirtyCards]);
   debug_time("Post Evacuate Cleanup 2", _cur_post_evacuate_cleanup_2_time_ms);
   if (G1CollectedHeap::heap()->evacuation_failed()) {
     debug_phase(_gc_par_phases[RecalculateUsed], 1);
@@ -485,6 +491,9 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set() const {
   if (G1CollectedHeap::heap()->should_do_eager_reclaim()) {
     debug_phase(_gc_par_phases[EagerlyReclaimHumongousObjects], 1);
   }
+  if (G1CollectedHeap::heap()->should_sample_collection_set_candidates()) {
+    debug_phase(_gc_par_phases[SampleCollectionSetCandidates], 1);
+  }
   debug_phase(_gc_par_phases[RedirtyCards], 1);
   debug_phase(_gc_par_phases[FreeCollectionSet], 1);
   trace_phase(_gc_par_phases[YoungFreeCSet], true, 1);

src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp

@@ -75,6 +75,7 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
     YoungFreeCSet,
     NonYoungFreeCSet,
     RebuildFreeList,
+    SampleCollectionSetCandidates,
     MergePSS,
     RemoveSelfForwardingPtr,
     ClearCardTable,
@@ -97,13 +98,24 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
     return GCParPhases(StrongOopStorageSetRoots + index);
   }
 
-  enum GCMergeRSWorkTimes {
-    MergeRSMergedSparse,
-    MergeRSMergedFine,
-    MergeRSMergedCoarse,
-    MergeRSDirtyCards
+  enum GCMergeRSWorkItems : uint {
+    MergeRSMergedInline = 0,
+    MergeRSMergedArrayOfCards,
+    MergeRSMergedHowl,
+    MergeRSMergedFull,
+    MergeRSHowlInline,
+    MergeRSHowlArrayOfCards,
+    MergeRSHowlBitmap,
+    MergeRSHowlFull,
+    MergeRSDirtyCards,
+    MergeRSContainersSentinel
   };
 
+  static constexpr const char* GCMergeRSWorkItemsStrings[MergeRSContainersSentinel] =
+    { "Merged Inline", "Merged ArrayOfCards", "Merged Howl", "Merged Full",
+      "Merged Howl Inline", "Merged Howl ArrayOfCards", "Merged Howl BitMap", "Merged Howl Full",
+      "Dirty Cards" };
+
   enum GCScanHRWorkItems {
     ScanHRScannedCards,
     ScanHRScannedBlocks,
@@ -173,6 +185,8 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
   double _recorded_young_cset_choice_time_ms;
   double _recorded_non_young_cset_choice_time_ms;
 
+  double _recorded_sample_collection_set_candidates_time_ms;
+
   double _recorded_preserve_cm_referents_time_ms;
 
   double _recorded_start_new_cset_time_ms;
@@ -329,6 +343,10 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
     _recorded_non_young_cset_choice_time_ms = time_ms;
   }
 
+  void record_sample_collection_set_candidates_time_ms(double time_ms) {
+    _recorded_sample_collection_set_candidates_time_ms = time_ms;
+  }
+
   void record_preserve_cm_referents_time_ms(double time_ms) {
     _recorded_preserve_cm_referents_time_ms = time_ms;
   }

src/hotspot/share/gc/g1/g1OopClosures.inline.hpp

@@ -32,7 +32,7 @@
 #include "gc/g1/g1ParScanThreadState.inline.hpp"
 #include "gc/g1/g1RemSet.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "memory/iterator.inline.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/compressedOops.inline.hpp"

src/hotspot/share/gc/g1/g1ParScanThreadState.hpp

@@ -30,7 +30,7 @@
 #include "gc/g1/g1RedirtyCardsQueue.hpp"
 #include "gc/g1/g1OopClosures.hpp"
 #include "gc/g1/g1RemSet.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/shared/ageTable.hpp"
 #include "gc/shared/partialArrayTaskStepper.hpp"
 #include "gc/shared/stringdedup/stringDedup.hpp"

src/hotspot/share/gc/g1/g1Policy.cpp

@@ -40,7 +40,7 @@
 #include "gc/g1/g1SurvivorRegions.hpp"
 #include "gc/g1/g1YoungGenSizer.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/shared/concurrentGCBreakpoints.hpp"
 #include "gc/shared/gcPolicyCounters.hpp"
 #include "logging/log.hpp"
@@ -50,6 +50,8 @@
 #include "utilities/growableArray.hpp"
 #include "utilities/pair.hpp"
 
+#include "gc/shared/gcTraceTime.inline.hpp"
+
 G1Policy::G1Policy(STWGCTimer* gc_timer) :
   _predictor(G1ConfidencePercent / 100.0),
   _analytics(new G1Analytics(&_predictor)),

src/hotspot/share/gc/g1/g1RemSet.cpp

@@ -26,6 +26,7 @@
 #include "gc/g1/g1BarrierSet.hpp"
 #include "gc/g1/g1BatchedGangTask.hpp"
 #include "gc/g1/g1BlockOffsetTable.inline.hpp"
+#include "gc/g1/g1CardSet.inline.hpp"
 #include "gc/g1/g1CardTable.inline.hpp"
 #include "gc/g1/g1CardTableEntryClosure.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
@@ -45,7 +46,6 @@
 #include "gc/g1/heapRegion.inline.hpp"
 #include "gc/g1/heapRegionManager.inline.hpp"
 #include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/sparsePRT.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
 #include "gc/shared/ptrQueue.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
@@ -381,9 +381,7 @@ public:
            "Trying to access index " SIZE_FORMAT " out of bounds " SIZE_FORMAT,
            card_idx >> _scan_chunks_shift, _num_total_scan_chunks);
     size_t const chunk_idx = card_idx >> _scan_chunks_shift;
-    if (!_region_scan_chunks[chunk_idx]) {
-      _region_scan_chunks[chunk_idx] = true;
-    }
+    _region_scan_chunks[chunk_idx] = true;
   }
 
   G1AbstractSubTask* create_cleanup_after_scan_heap_roots_task() {
@@ -1061,9 +1059,9 @@ void G1RemSet::prepare_for_scan_heap_roots() {
   _scan_state->prepare();
 }
 
-// Small ring buffer used for prefetching cards for read/write from the card
+// Small ring buffer used for prefetching cards for write from the card
 // table during GC.
-template <class T, bool for_write>
+template <class T>
 class G1MergeHeapRootsPrefetchCache {
 public:
   static const uint CacheSize = G1MergeHeapRootsPrefetchCacheSize;
@@ -1096,11 +1094,7 @@ public:
   }
 
   T* push(T* elem) {
-    if (for_write) {
-      Prefetch::write(elem, 0);
-    } else {
-      Prefetch::read(elem, 0);
-    }
+    Prefetch::write(elem, 0);
     T* result = _cache[_cur_cache_idx];
     _cache[_cur_cache_idx++] = elem;
     _cur_cache_idx &= (CacheSize - 1);
@@ -1111,7 +1105,33 @@ public:
 
 class G1MergeHeapRootsTask : public AbstractGangTask {
 
-  // Visitor for remembered sets, dropping entries onto the card table.
+  class G1MergeCardSetStats {
+    size_t _merged[G1GCPhaseTimes::MergeRSContainersSentinel];
+
+  public:
+    G1MergeCardSetStats() {
+      for (uint i = 0; i < ARRAY_SIZE(_merged); i++) {
+        _merged[i] = 0;
+      }
+    }
+
+    void inc_card_set_merged(uint tag) {
+      assert(tag < ARRAY_SIZE(_merged), "tag out of bounds %u", tag);
+      _merged[tag]++;
+    }
+
+    void inc_cards_dirty(size_t increment = 1) {
+      _merged[G1GCPhaseTimes::MergeRSDirtyCards] += increment;
+    }
+
+    size_t merged(uint i) const { return _merged[i]; }
+  };
+
+  // Visitor for remembered sets. Several methods of it are called by a region's
+  // card set iterator to drop card set remembered set entries onto the card.
+  // table. This is in addition to being the HeapRegionClosure to iterate over
+  // all region's remembered sets.
+  //
   // We add a small prefetching cache in front of the actual work as dropping
   // onto the card table is basically random memory access. This improves
   // performance of this operation significantly.
@@ -1121,24 +1141,20 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
     G1RemSetScanState* _scan_state;
     G1CardTable* _ct;
 
-    uint _merged_sparse;
-    uint _merged_fine;
-    uint _merged_coarse;
-
-    size_t _cards_dirty;
+    G1MergeCardSetStats _stats;
 
     // Cached card table index of the currently processed region to avoid constant
     // recalculation as our remembered set containers are per region.
     size_t _region_base_idx;
 
-    class G1MergeCardSetCache : public G1MergeHeapRootsPrefetchCache<G1CardTable::CardValue, true> {
+    class G1MergeCardSetCache : public G1MergeHeapRootsPrefetchCache<G1CardTable::CardValue> {
       G1MergeCardSetClosure* const _merge_card_cl;
 
     public:
       G1MergeCardSetCache(G1MergeCardSetClosure* const merge_card_cl) :
         // Initially set dummy card value to Dirty to avoid any actual mark work if we
         // try to process it.
-        G1MergeHeapRootsPrefetchCache<G1CardTable::CardValue, true>(G1CardTable::dirty_card_val()),
+        G1MergeHeapRootsPrefetchCache<G1CardTable::CardValue>(G1CardTable::dirty_card_val()),
         _merge_card_cl(merge_card_cl) { }
 
       ~G1MergeCardSetCache() {
@@ -1160,26 +1176,19 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
 
     void mark_card(G1CardTable::CardValue* value) {
       if (_ct->mark_clean_as_dirty(value)) {
-        _cards_dirty++;
+        _stats.inc_cards_dirty();
         _scan_state->set_chunk_dirty(_ct->index_for_cardvalue(value));
       }
     }
 
-    void start_iterate(uint const region_idx) {
-      _region_base_idx = (size_t)region_idx << HeapRegion::LogCardsPerRegion;
-    }
-
   public:
+
     G1MergeCardSetClosure(G1RemSetScanState* scan_state) :
       _scan_state(scan_state),
       _ct(G1CollectedHeap::heap()->card_table()),
-      _merged_sparse(0),
-      _merged_fine(0),
-      _merged_coarse(0),
-      _cards_dirty(0),
+      _stats(),
       _region_base_idx(0),
-      _merge_card_set_cache(this) {
-    }
+      _merge_card_set_cache(this) { }
 
     void do_card(uint const card_idx) {
       G1CardTable::CardValue* to_prefetch = _ct->byte_for_index(_region_base_idx + card_idx);
@@ -1188,47 +1197,26 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
       mark_card(to_process);
     }
 
-    void next_coarse_prt(uint const region_idx) {
-      if (!remember_if_interesting(region_idx)) {
-        return;
+    // Returns whether the given region actually needs iteration.
+    bool start_iterate(uint const tag, uint const region_idx) {
+      assert(tag < G1GCPhaseTimes::MergeRSDirtyCards, "invalid tag %u", tag);
+      if (remember_if_interesting(region_idx)) {
+        _region_base_idx = (size_t)region_idx << HeapRegion::LogCardsPerRegion;
+        _stats.inc_card_set_merged(tag);
+        return true;
       }
+      return false;
+    }
 
-      _merged_coarse++;
-
-      start_iterate(region_idx);
-      _cards_dirty += _ct->mark_region_dirty(_region_base_idx, HeapRegion::CardsPerRegion);
+    void do_card_range(uint const start_card_idx, uint const length) {
+      assert(start_card_idx == 0, "must be");
+      assert(length == HeapRegion::CardsPerRegion, "must be");
+      size_t num_dirtied = _ct->mark_range_dirty(_region_base_idx, HeapRegion::CardsPerRegion);
+      _stats.inc_cards_dirty(num_dirtied);
       _scan_state->set_chunk_region_dirty(_region_base_idx);
     }
 
-    void next_fine_prt(uint const region_idx, BitMap* bm) {
-      if (!remember_if_interesting(region_idx)) {
-        return;
-      }
-
-      _merged_fine++;
-
-      start_iterate(region_idx);
-      BitMap::idx_t cur = bm->get_next_one_offset(0);
-      while (cur != bm->size()) {
-        do_card((uint)cur);
-        cur = bm->get_next_one_offset(cur + 1);
-      }
-    }
-
-    void next_sparse_prt(uint const region_idx, SparsePRTEntry::card_elem_t* cards, uint const num_cards) {
-      if (!remember_if_interesting(region_idx)) {
-        return;
-      }
-
-      _merged_sparse++;
-
-      start_iterate(region_idx);
-      for (uint i = 0; i < num_cards; i++) {
-        do_card(cards[i]);
-      }
-    }
-
-    // Helper to put the remembered set cards for these regions onto the card
+    // Helper to merge the cards in the card set for the given region onto the card
     // table.
     //
     // Called directly for humongous starts regions because we should not add
@@ -1242,12 +1230,12 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
     // pass) or the "dirty" list will be merged with the "all" list later otherwise.
     // (And there is no problem either way if the region does not contain dirty
     // cards).
-    void dump_rem_set_for_region(HeapRegion* r) {
+    void merge_card_set_for_region(HeapRegion* r) {
       assert(r->in_collection_set() || r->is_starts_humongous(), "must be");
 
       HeapRegionRemSet* rem_set = r->rem_set();
       if (!rem_set->is_empty()) {
-        rem_set->iterate_prts(*this);
+        rem_set->iterate_for_merge(*this);
       }
     }
 
@@ -1255,25 +1243,22 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
       assert(r->in_collection_set(), "must be");
 
       _scan_state->add_all_dirty_region(r->hrm_index());
-      dump_rem_set_for_region(r);
+      merge_card_set_for_region(r);
 
       return false;
     }
 
-    size_t merged_sparse() const { return _merged_sparse; }
-    size_t merged_fine() const { return _merged_fine; }
-    size_t merged_coarse() const { return _merged_coarse; }
-
-    size_t cards_dirty() const { return _cards_dirty; }
+    G1MergeCardSetStats stats() const { return _stats; }
   };
 
   // Visitor for the remembered sets of humongous candidate regions to merge their
   // remembered set into the card table.
   class G1FlushHumongousCandidateRemSets : public HeapRegionClosure {
-    G1MergeCardSetClosure _cl;
+    G1RemSetScanState* _scan_state;
+    G1MergeCardSetStats _merge_stats;
 
   public:
-    G1FlushHumongousCandidateRemSets(G1RemSetScanState* scan_state) : _cl(scan_state) { }
+    G1FlushHumongousCandidateRemSets(G1RemSetScanState* scan_state) : _scan_state(scan_state), _merge_stats() { }
 
     virtual bool do_heap_region(HeapRegion* r) {
       G1CollectedHeap* g1h = G1CollectedHeap::heap();
@@ -1287,7 +1272,12 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
       guarantee(r->rem_set()->occupancy_less_or_equal_than(G1EagerReclaimRemSetThreshold),
                 "Found a not-small remembered set here. This is inconsistent with previous assumptions.");
 
-      _cl.dump_rem_set_for_region(r);
+      G1MergeCardSetStats stats;
+      {
+        G1MergeCardSetClosure cl(_scan_state);
+        cl.merge_card_set_for_region(r);
+        stats = cl.stats();
+      }
 
       // We should only clear the card based remembered set here as we will not
       // implicitly rebuild anything else during eager reclaim. Note that at the moment
@@ -1307,16 +1297,11 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
       return false;
     }
 
-    size_t merged_sparse() const { return _cl.merged_sparse(); }
-    size_t merged_fine() const { return _cl.merged_fine(); }
-    size_t merged_coarse() const { return _cl.merged_coarse(); }
-
-    size_t cards_dirty() const { return _cl.cards_dirty(); }
+    size_t merged(uint i) const { return _merge_stats.merged(i); }
   };
 
   // Visitor for the log buffer entries to merge them into the card table.
   class G1MergeLogBufferCardsClosure : public G1CardTableEntryClosure {
-    friend class G1MergeLogBufferCardsCache;
 
     G1RemSetScanState* _scan_state;
     G1CardTable* _ct;
@@ -1324,23 +1309,6 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
     size_t _cards_dirty;
     size_t _cards_skipped;
 
-    class G1MergeLogBufferCardsCache : public G1MergeHeapRootsPrefetchCache<G1CardTable::CardValue, false> {
-      G1MergeLogBufferCardsClosure* const _merge_log_buffer_cl;
-
-    public:
-      G1MergeLogBufferCardsCache(G1MergeLogBufferCardsClosure* const merge_log_buffer_cl) :
-        // Initially set dummy card value to Clean to avoid any actual work if we
-        // try to process it.
-        G1MergeHeapRootsPrefetchCache<G1CardTable::CardValue, false>(G1CardTable::clean_card_val()),
-        _merge_log_buffer_cl(merge_log_buffer_cl) { }
-
-      ~G1MergeLogBufferCardsCache() {
-        for (uint i = 0; i < CacheSize; i++) {
-          _merge_log_buffer_cl->process_card(push(&_dummy_card));
-        }
-      }
-    } _merge_log_buffer_cache;
-
     void process_card(CardValue* card_ptr) {
       if (*card_ptr == G1CardTable::dirty_card_val()) {
         uint const region_idx = _ct->region_idx_for(card_ptr);
@@ -1355,8 +1323,7 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
       _scan_state(scan_state),
       _ct(g1h->card_table()),
       _cards_dirty(0),
-      _cards_skipped(0),
-      _merge_log_buffer_cache(this)
+      _cards_skipped(0)
     {}
 
     void do_card_ptr(CardValue* card_ptr, uint worker_id) {
@@ -1373,8 +1340,7 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
       // This code may count duplicate entries in the log buffers (even if rare) multiple
       // times.
       if (_scan_state->contains_cards_to_process(region_idx)) {
-        CardValue* to_process = _merge_log_buffer_cache.push(card_ptr);
-        process_card(to_process);
+        process_card(card_ptr);
       } else {
         // We may have had dirty cards in the (initial) collection set (or the
         // young regions which are always in the initial collection set). We do
@@ -1442,22 +1408,24 @@ public:
       G1FlushHumongousCandidateRemSets cl(_scan_state);
       g1h->heap_region_iterate(&cl);
 
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged_sparse(), G1GCPhaseTimes::MergeRSMergedSparse);
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged_fine(), G1GCPhaseTimes::MergeRSMergedFine);
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged_coarse(), G1GCPhaseTimes::MergeRSMergedCoarse);
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.cards_dirty(), G1GCPhaseTimes::MergeRSDirtyCards);
+      for (uint i = 0; i < G1GCPhaseTimes::MergeRSContainersSentinel; i++) {
+        p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged(i), i);
+      }
     }
 
     // Merge remembered sets of current candidates.
     {
       G1GCParPhaseTimesTracker x(p, merge_remset_phase, worker_id, _initial_evacuation /* must_record */);
-      G1MergeCardSetClosure cl(_scan_state);
-      g1h->collection_set_iterate_increment_from(&cl, &_hr_claimer, worker_id);
+      G1MergeCardSetStats stats;
+      {
+        G1MergeCardSetClosure cl(_scan_state);
+        g1h->collection_set_iterate_increment_from(&cl, &_hr_claimer, worker_id);
+        stats = cl.stats();
+      }
 
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged_sparse(), G1GCPhaseTimes::MergeRSMergedSparse);
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged_fine(), G1GCPhaseTimes::MergeRSMergedFine);
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged_coarse(), G1GCPhaseTimes::MergeRSMergedCoarse);
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.cards_dirty(), G1GCPhaseTimes::MergeRSDirtyCards);
+      for (uint i = 0; i < G1GCPhaseTimes::MergeRSContainersSentinel; i++) {
+        p->record_or_add_thread_work_item(merge_remset_phase, worker_id, stats.merged(i), i);
+      }
     }
 
     // Apply closure to log entries in the HCC.
@@ -1486,20 +1454,25 @@ public:
 };
 
 void G1RemSet::print_merge_heap_roots_stats() {
-  size_t num_visited_cards = _scan_state->num_visited_cards();
+  LogTarget(Debug, gc, remset) lt;
+  if (lt.is_enabled()) {
+    LogStream ls(lt);
 
-  size_t total_dirty_region_cards = _scan_state->num_cards_in_dirty_regions();
+    size_t num_visited_cards = _scan_state->num_visited_cards();
 
-  G1CollectedHeap* g1h = G1CollectedHeap::heap();
-  size_t total_old_region_cards =
-    (g1h->num_regions() - (g1h->num_free_regions() - g1h->collection_set()->cur_length())) * HeapRegion::CardsPerRegion;
+    size_t total_dirty_region_cards = _scan_state->num_cards_in_dirty_regions();
 
-  log_debug(gc,remset)("Visited cards " SIZE_FORMAT " Total dirty " SIZE_FORMAT " (%.2lf%%) Total old " SIZE_FORMAT " (%.2lf%%)",
-                       num_visited_cards,
-                       total_dirty_region_cards,
-                       percent_of(num_visited_cards, total_dirty_region_cards),
-                       total_old_region_cards,
-                       percent_of(num_visited_cards, total_old_region_cards));
+    G1CollectedHeap* g1h = G1CollectedHeap::heap();
+    size_t total_old_region_cards =
+      (g1h->num_regions() - (g1h->num_free_regions() - g1h->collection_set()->cur_length())) * HeapRegion::CardsPerRegion;
+
+    ls.print_cr("Visited cards " SIZE_FORMAT " Total dirty " SIZE_FORMAT " (%.2lf%%) Total old " SIZE_FORMAT " (%.2lf%%)",
+                num_visited_cards,
+                total_dirty_region_cards,
+                percent_of(num_visited_cards, total_dirty_region_cards),
+                total_old_region_cards,
+                percent_of(num_visited_cards, total_old_region_cards));
+  }
 }
 
 void G1RemSet::merge_heap_roots(bool initial_evacuation) {
@@ -1531,9 +1504,7 @@ void G1RemSet::merge_heap_roots(bool initial_evacuation) {
     workers->run_task(&cl, num_workers);
   }
 
-  if (log_is_enabled(Debug, gc, remset)) {
-    print_merge_heap_roots_stats();
-  }
+  print_merge_heap_roots_stats();
 }
 
 void G1RemSet::complete_evac_phase(bool has_more_than_one_evacuation_phase) {
@@ -1548,6 +1519,15 @@ G1AbstractSubTask* G1RemSet::create_cleanup_after_scan_heap_roots_task() {
   return _scan_state->create_cleanup_after_scan_heap_roots_task();
 }
 
+void G1RemSet::print_coarsen_stats() {
+  LogTarget(Debug, gc, remset) lt;
+  if (lt.is_enabled()) {
+    LogStream ls(lt);
+
+    G1CardSet::print_coarsen_stats(&ls);
+  }
+}
+
 inline void check_card_ptr(CardTable::CardValue* card_ptr, G1CardTable* ct) {
 #ifdef ASSERT
   G1CollectedHeap* g1h = G1CollectedHeap::heap();

src/hotspot/share/gc/g1/g1RemSet.hpp

@@ -107,6 +107,10 @@ public:
   // Prepare for and cleanup after scanning the heap roots. Must be called
   // once before and after in sequential code.
   void prepare_for_scan_heap_roots();
+  // Cleans the card table from temporary duplicate detection information.
+  void cleanup_after_scan_heap_roots();
+  // Print coarsening stats.
+  void print_coarsen_stats();
   // Creates a gang task for cleaining up temporary data structures and the
   // card table, removing temporary duplicate detection information.
   G1AbstractSubTask* create_cleanup_after_scan_heap_roots_task();

src/hotspot/share/gc/g1/g1RemSetSummary.cpp

@@ -24,6 +24,7 @@
 
 #include "precompiled.hpp"
 #include "gc/g1/g1BarrierSet.hpp"
+#include "gc/g1/g1CardSetMemory.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1ConcurrentRefine.hpp"
 #include "gc/g1/g1ConcurrentRefineThread.hpp"
@@ -31,7 +32,7 @@
 #include "gc/g1/g1RemSet.hpp"
 #include "gc/g1/g1RemSetSummary.hpp"
 #include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "memory/allocation.inline.hpp"
 #include "memory/iterator.hpp"
 #include "runtime/thread.inline.hpp"
@@ -48,9 +49,10 @@ void G1RemSetSummary::update() {
       _counter++;
     }
   } collector(this);
+
   G1CollectedHeap* g1h = G1CollectedHeap::heap();
   g1h->concurrent_refine()->threads_do(&collector);
-  _num_coarsenings = HeapRegionRemSet::n_coarsenings();
+  _coarsenings = HeapRegionRemSet::coarsen_stats();
 
   set_sampling_task_vtime(g1h->rem_set()->sampling_task_vtime());
 }
@@ -68,7 +70,7 @@ double G1RemSetSummary::rs_thread_vtime(uint thread) const {
 }
 
 G1RemSetSummary::G1RemSetSummary(bool should_update) :
-  _num_coarsenings(0),
+  _coarsenings(),
   _num_vtimes(G1ConcurrentRefine::max_num_threads()),
   _rs_threads_vtimes(NEW_C_HEAP_ARRAY(double, _num_vtimes, mtGC)),
   _sampling_task_vtime(0.0f) {
@@ -88,7 +90,7 @@ void G1RemSetSummary::set(G1RemSetSummary* other) {
   assert(other != NULL, "just checking");
   assert(_num_vtimes == other->_num_vtimes, "just checking");
 
-  _num_coarsenings = other->num_coarsenings();
+  _coarsenings = other->_coarsenings;
 
   memcpy(_rs_threads_vtimes, other->_rs_threads_vtimes, sizeof(double) * _num_vtimes);
 
@@ -99,7 +101,7 @@ void G1RemSetSummary::subtract_from(G1RemSetSummary* other) {
   assert(other != NULL, "just checking");
   assert(_num_vtimes == other->_num_vtimes, "just checking");
 
-  _num_coarsenings = other->num_coarsenings() - _num_coarsenings;
+  _coarsenings.subtract_from(other->_coarsenings);
 
   for (uint i = 0; i < _num_vtimes; i++) {
     set_rs_thread_vtime(i, other->rs_thread_vtime(i) - rs_thread_vtime(i));
@@ -112,9 +114,11 @@ class RegionTypeCounter {
 private:
   const char* _name;
 
+  size_t _rs_wasted_mem_size;
   size_t _rs_mem_size;
   size_t _cards_occupied;
   size_t _amount;
+  size_t _amount_tracked;
 
   size_t _code_root_mem_size;
   size_t _code_root_elems;
@@ -136,21 +140,25 @@ private:
   }
 
   size_t amount() const { return _amount; }
+  size_t amount_tracked() const { return _amount_tracked; }
 
 public:
 
-  RegionTypeCounter(const char* name) : _name(name), _rs_mem_size(0), _cards_occupied(0),
-    _amount(0), _code_root_mem_size(0), _code_root_elems(0) { }
+  RegionTypeCounter(const char* name) : _name(name), _rs_wasted_mem_size(0), _rs_mem_size(0), _cards_occupied(0),
+    _amount(0), _amount_tracked(0), _code_root_mem_size(0), _code_root_elems(0) { }
 
-  void add(size_t rs_mem_size, size_t cards_occupied, size_t code_root_mem_size,
-    size_t code_root_elems) {
+  void add(size_t rs_wasted_mem_size, size_t rs_mem_size, size_t cards_occupied,
+           size_t code_root_mem_size, size_t code_root_elems, bool tracked) {
+    _rs_wasted_mem_size += rs_wasted_mem_size;
     _rs_mem_size += rs_mem_size;
     _cards_occupied += cards_occupied;
     _code_root_mem_size += code_root_mem_size;
     _code_root_elems += code_root_elems;
     _amount++;
+    _amount_tracked += tracked ? 1 : 0;
   }
 
+  size_t rs_wasted_mem_size() const { return _rs_wasted_mem_size; }
   size_t rs_mem_size() const { return _rs_mem_size; }
   size_t cards_occupied() const { return _cards_occupied; }
 
@@ -158,15 +166,18 @@ public:
   size_t code_root_elems() const { return _code_root_elems; }
 
   void print_rs_mem_info_on(outputStream * out, size_t total) {
-    out->print_cr("    " SIZE_FORMAT_W(8) "%s (%5.1f%%) by " SIZE_FORMAT " %s regions",
-        byte_size_in_proper_unit(rs_mem_size()),
-        proper_unit_for_byte_size(rs_mem_size()),
-        rs_mem_size_percent_of(total), amount(), _name);
+    out->print_cr("    " SIZE_FORMAT_W(8) " (%5.1f%%) by " SIZE_FORMAT " "
+                  "(" SIZE_FORMAT ") %s regions wasted " SIZE_FORMAT,
+                  rs_mem_size(), rs_mem_size_percent_of(total),
+                  amount_tracked(), amount(),
+                  _name, rs_wasted_mem_size());
   }
 
   void print_cards_occupied_info_on(outputStream * out, size_t total) {
-    out->print_cr("     " SIZE_FORMAT_W(8) " (%5.1f%%) entries by " SIZE_FORMAT " %s regions",
-        cards_occupied(), cards_occupied_percent_of(total), amount(), _name);
+    out->print_cr("     " SIZE_FORMAT_W(8) " (%5.1f%%) entries by " SIZE_FORMAT " "
+                  "(" SIZE_FORMAT ") %s regions",
+                  cards_occupied(), cards_occupied_percent_of(total),
+                  amount_tracked(), amount(), _name);
   }
 
   void print_code_root_mem_info_on(outputStream * out, size_t total) {
@@ -195,6 +206,7 @@ private:
   size_t _max_rs_mem_sz;
   HeapRegion* _max_rs_mem_sz_region;
 
+  size_t total_rs_wasted_mem_sz() const     { return _all.rs_wasted_mem_size(); }
   size_t total_rs_mem_sz() const            { return _all.rs_mem_size(); }
   size_t total_cards_occupied() const       { return _all.cards_occupied(); }
 
@@ -222,6 +234,7 @@ public:
 
     // HeapRegionRemSet::mem_size() includes the
     // size of the strong code roots
+    size_t rs_wasted_mem_sz = hrrs->wasted_mem_size();
     size_t rs_mem_sz = hrrs->mem_size();
     if (rs_mem_sz > _max_rs_mem_sz) {
       _max_rs_mem_sz = rs_mem_sz;
@@ -249,8 +262,10 @@ public:
     } else {
       ShouldNotReachHere();
     }
-    current->add(rs_mem_sz, occupied_cards, code_root_mem_sz, code_root_elems);
-    _all.add(rs_mem_sz, occupied_cards, code_root_mem_sz, code_root_elems);
+    current->add(rs_wasted_mem_sz, rs_mem_sz, occupied_cards,
+                 code_root_mem_sz, code_root_elems, r->rem_set()->is_tracked());
+    _all.add(rs_wasted_mem_sz, rs_mem_sz, occupied_cards,
+             code_root_mem_sz, code_root_elems, r->rem_set()->is_tracked());
 
     return false;
   }
@@ -259,23 +274,15 @@ public:
     RegionTypeCounter* counters[] = { &_young, &_humongous, &_free, &_old, &_archive, NULL };
 
     out->print_cr(" Current rem set statistics");
-    out->print_cr("  Total per region rem sets sizes = " SIZE_FORMAT "%s."
-                  " Max = " SIZE_FORMAT "%s.",
-                  byte_size_in_proper_unit(total_rs_mem_sz()),
-                  proper_unit_for_byte_size(total_rs_mem_sz()),
-                  byte_size_in_proper_unit(max_rs_mem_sz()),
-                  proper_unit_for_byte_size(max_rs_mem_sz()));
+    out->print_cr("  Total per region rem sets sizes = " SIZE_FORMAT
+                  " Max = " SIZE_FORMAT " wasted = " SIZE_FORMAT,
+                  total_rs_mem_sz(),
+                  max_rs_mem_sz(),
+                  total_rs_wasted_mem_sz());
     for (RegionTypeCounter** current = &counters[0]; *current != NULL; current++) {
       (*current)->print_rs_mem_info_on(out, total_rs_mem_sz());
     }
 
-    out->print_cr("   Static structures = " SIZE_FORMAT "%s,"
-                  " free_lists = " SIZE_FORMAT "%s.",
-                  byte_size_in_proper_unit(HeapRegionRemSet::static_mem_size()),
-                  proper_unit_for_byte_size(HeapRegionRemSet::static_mem_size()),
-                  byte_size_in_proper_unit(HeapRegionRemSet::fl_mem_size()),
-                  proper_unit_for_byte_size(HeapRegionRemSet::fl_mem_size()));
-
     out->print_cr("    " SIZE_FORMAT " occupied cards represented.",
                   total_cards_occupied());
     for (RegionTypeCounter** current = &counters[0]; *current != NULL; current++) {
@@ -285,12 +292,14 @@ public:
     // Largest sized rem set region statistics
     HeapRegionRemSet* rem_set = max_rs_mem_sz_region()->rem_set();
     out->print_cr("    Region with largest rem set = " HR_FORMAT ", "
-                  "size = " SIZE_FORMAT "%s, occupied = " SIZE_FORMAT "%s.",
+                  "size = " SIZE_FORMAT " occupied = " SIZE_FORMAT,
                   HR_FORMAT_PARAMS(max_rs_mem_sz_region()),
-                  byte_size_in_proper_unit(rem_set->mem_size()),
-                  proper_unit_for_byte_size(rem_set->mem_size()),
-                  byte_size_in_proper_unit(rem_set->occupied()),
-                  proper_unit_for_byte_size(rem_set->occupied()));
+                  rem_set->mem_size(),
+                  rem_set->occupied());
+
+    HeapRegionRemSet::print_static_mem_size(out);
+    G1CardSetFreePool::free_list_pool()->print_on(out);
+
     // Strong code root statistics
     HeapRegionRemSet* max_code_root_rem_set = max_code_root_mem_sz_region()->rem_set();
     out->print_cr("  Total heap region code root sets sizes = " SIZE_FORMAT "%s."
@@ -319,7 +328,8 @@ public:
 };
 
 void G1RemSetSummary::print_on(outputStream* out) {
-  out->print_cr("  Did " SIZE_FORMAT " coarsenings.", num_coarsenings());
+  out->print("Coarsening: ");
+  _coarsenings.print_on(out);
   out->print_cr("  Concurrent refinement threads times (s)");
   out->print("     ");
   for (uint i = 0; i < _num_vtimes; i++) {

src/hotspot/share/gc/g1/g1RemSetSummary.hpp

@@ -25,6 +25,7 @@
 #ifndef SHARE_GC_G1_G1REMSETSUMMARY_HPP
 #define SHARE_GC_G1_G1REMSETSUMMARY_HPP
 
+#include "gc/g1/g1CardSet.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/ostream.hpp"
 
@@ -33,7 +34,7 @@ class G1RemSet;
 // A G1RemSetSummary manages statistical information about the G1RemSet
 
 class G1RemSetSummary {
-  size_t _num_coarsenings;
+  G1CardSetCoarsenStats _coarsenings;
 
   size_t _num_vtimes;
   double* _rs_threads_vtimes;
@@ -65,10 +66,6 @@ public:
   double sampling_task_vtime() const {
     return _sampling_task_vtime;
   }
-
-  size_t num_coarsenings() const {
-    return _num_coarsenings;
-  }
 };
 
 #endif // SHARE_GC_G1_G1REMSETSUMMARY_HPP

src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp

@@ -26,7 +26,7 @@
 #include "gc/g1/g1CollectionSetChooser.hpp"
 #include "gc/g1/g1RemSetTrackingPolicy.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "runtime/safepoint.hpp"
 
 bool G1RemSetTrackingPolicy::needs_scan_for_rebuild(HeapRegion* r) const {

src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp

@@ -25,8 +25,10 @@
 #include "precompiled.hpp"
 
 #include "compiler/oopMap.hpp"
+#include "gc/g1/g1CardSetMemory.hpp"
 #include "gc/g1/g1CardTableEntryClosure.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1CollectionSetCandidates.hpp"
 #include "gc/g1/g1ConcurrentMark.inline.hpp"
 #include "gc/g1/g1EvacStats.inline.hpp"
 #include "gc/g1/g1ParScanThreadState.hpp"
@@ -41,6 +43,9 @@ G1PostEvacuateCollectionSetCleanupTask1::G1PostEvacuateCollectionSetCleanupTask1
 {
   add_serial_task(new MergePssTask(per_thread_states));
   add_serial_task(new RecalculateUsedTask());
+  if (SampleCollectionSetCandidatesTask::should_execute()) {
+    add_serial_task(new SampleCollectionSetCandidatesTask());
+  }
   if (RemoveSelfForwardPtrsTask::should_execute()) {
     add_parallel_task(new RemoveSelfForwardPtrsTask(rdcqs));
   }
@@ -64,6 +69,32 @@ void G1PostEvacuateCollectionSetCleanupTask1::RecalculateUsedTask::do_work(uint
   G1CollectedHeap::heap()->update_used_after_gc();
 }
 
+bool G1PostEvacuateCollectionSetCleanupTask1::SampleCollectionSetCandidatesTask::should_execute() {
+  return G1CollectedHeap::heap()->should_sample_collection_set_candidates();
+}
+
+double G1PostEvacuateCollectionSetCleanupTask1::SampleCollectionSetCandidatesTask::worker_cost() const {
+  return should_execute() ? 1.0 : AlmostNoWork;
+}
+
+class G1SampleCollectionSetCandidatesClosure : public HeapRegionClosure {
+public:
+  G1CardSetMemoryStats _total;
+
+  bool do_heap_region(HeapRegion* r) override {
+    _total.add(r->rem_set()->card_set_memory_stats());
+    return false;
+  }
+};
+
+void G1PostEvacuateCollectionSetCleanupTask1::SampleCollectionSetCandidatesTask::do_work(uint worker_id) {
+  G1CollectedHeap* g1h = G1CollectedHeap::heap();
+
+  G1SampleCollectionSetCandidatesClosure cl;
+  g1h->collection_set()->candidates()->iterate(&cl);
+  g1h->set_collection_set_candidates_stats(cl._total);
+}
+
 bool G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask::should_execute() {
   return G1CollectedHeap::heap()->evacuation_failed();
 }

src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp

@@ -38,11 +38,13 @@ class G1RedirtyCardsQueueSet;
 // First set of post evacuate collection set tasks containing ("s" means serial):
 // - Merge PSS (s)
 // - Recalculate Used (s)
+// - Sample Collection Set Candidates (s)
 // - Remove Self Forwards (on evacuation failure)
 // - Clear Card Table
 class G1PostEvacuateCollectionSetCleanupTask1 : public G1BatchedGangTask {
   class MergePssTask;
   class RecalculateUsedTask;
+  class SampleCollectionSetCandidatesTask;
   class RemoveSelfForwardPtrsTask;
 
 public:
@@ -68,6 +70,16 @@ public:
   void do_work(uint worker_id) override;
 };
 
+class G1PostEvacuateCollectionSetCleanupTask1::SampleCollectionSetCandidatesTask : public G1AbstractSubTask {
+public:
+  SampleCollectionSetCandidatesTask() : G1AbstractSubTask(G1GCPhaseTimes::SampleCollectionSetCandidates) { }
+
+  static bool should_execute();
+
+  double worker_cost() const override;
+  void do_work(uint worker_id) override;
+};
+
 class G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask : public G1AbstractSubTask {
   G1ParRemoveSelfForwardPtrsTask _task;
 

src/hotspot/share/gc/g1/g1_globals.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -130,15 +130,16 @@
           range(0, max_jint)                                                \
                                                                             \
   product(size_t, G1ConcRefinementThresholdStep, 2,                         \
-          "Each time the rset update queue increases by this amount "       \
-          "activate the next refinement thread if available. "              \
+          "Each time the remembered set update queue increases by this "    \
+          "amount activate the next refinement thread if available. "       \
           "The actual step size will be selected ergonomically by "         \
           "default, with this value used to determine a lower bound.")      \
           range(1, SIZE_MAX)                                                \
                                                                             \
   product(intx, G1RSetUpdatingPauseTimePercent, 10,                         \
           "A target percentage of time that is allowed to be spend on "     \
-          "process RS update buffers during the collection pause.")         \
+          "processing remembered set update buffers during the collection " \
+          "pause.")                                                         \
           range(0, 100)                                                     \
                                                                             \
   product(bool, G1UseAdaptiveConcRefinement, true,                          \
@@ -153,26 +154,40 @@
           "The threshold that defines (>=) a hot card.")                    \
           range(0, max_jubyte)                                              \
                                                                             \
-  develop(intx, G1RSetRegionEntriesBase, 256,                               \
-          "Max number of regions in a fine-grain table per MB.")            \
-          range(1, max_jint/wordSize)                                       \
+  develop(uint, G1RemSetArrayOfCardsEntriesBase, 4,                         \
+          "Maximum number of entries per region in the Array of Cards "     \
+          "card set container per MB of a heap region.")                    \
+          range(1, 65536)                                                   \
                                                                             \
-  product(intx, G1RSetRegionEntries, 0,                                     \
-          "Max number of regions for which we keep bitmaps."                \
-          "Will be set ergonomically by default")                           \
-          range(0, max_jint/wordSize)                                       \
-          constraint(G1RSetRegionEntriesConstraintFunc,AfterErgo)           \
+  product(uint, G1RemSetArrayOfCardsEntries, 0,  EXPERIMENTAL,              \
+          "Maximum number of entries per Array of Cards card set "          \
+          "container. Will be set ergonomically by default.")               \
+          range(0, 65536)                                                   \
+          constraint(G1RemSetArrayOfCardsEntriesConstraintFunc,AfterErgo)   \
                                                                             \
-  develop(intx, G1RSetSparseRegionEntriesBase, 4,                           \
-          "Max number of entries per region in a sparse table "             \
-          "per MB.")                                                        \
-          range(1, max_jint/wordSize)                                       \
+  product(uint, G1RemSetHowlMaxNumBuckets, 8, EXPERIMENTAL,                 \
+          "Maximum number of buckets per Howl card set container. The "     \
+          "default gives at worst bitmaps of size 8k. This showed to be a " \
+          "good tradeoff between bitmap size (waste) and cacheability of "  \
+          "the bucket array. Must be a power of two.")                      \
+          range(1, 1024)                                                    \
+          constraint(G1RemSetHowlMaxNumBucketsConstraintFunc,AfterErgo)     \
                                                                             \
-  product(intx, G1RSetSparseRegionEntries, 0,                               \
-          "Max number of entries per region in a sparse table."             \
-          "Will be set ergonomically by default.")                          \
-          range(0, max_jint/wordSize)                                       \
-          constraint(G1RSetSparseRegionEntriesConstraintFunc,AfterErgo)     \
+  product(uint, G1RemSetHowlNumBuckets, 0, EXPERIMENTAL,                    \
+          "Number of buckets per Howl card set container. Must be a power " \
+          "of two. Will be set ergonomically by default.")                  \
+          range(0, 1024)                                                    \
+          constraint(G1RemSetHowlNumBucketsConstraintFunc,AfterErgo)        \
+                                                                            \
+  product(uint, G1RemSetCoarsenHowlBitmapToHowlFullPercent, 90, EXPERIMENTAL, \
+          "Percentage at which to coarsen a Howl bitmap to Howl full card " \
+          "set container.")                                                 \
+          range(1, 100)                                                     \
+                                                                            \
+  product(uint, G1RemSetCoarsenHowlToFullPercent, 90, EXPERIMENTAL,         \
+          "Percentage at which to coarsen a Howl card set to Full card "    \
+          "set container.")                                                 \
+          range(1, 100)                                                     \
                                                                             \
   develop(intx, G1MaxVerifyFailures, -1,                                    \
           "The maximum number of verification failures to print.  "         \
@@ -190,7 +205,7 @@
           constraint(G1HeapRegionSizeConstraintFunc,AfterMemoryInit)        \
                                                                             \
   product(uint, G1ConcRefinementThreads, 0,                                 \
-          "The number of parallel rem set update threads. "                 \
+          "The number of parallel remembered set update threads. "          \
           "Will be set ergonomically by default.")                          \
           range(0, (max_jint-1)/wordSize)                                   \
                                                                             \
@@ -309,7 +324,23 @@
           "disables this check.")                                           \
           range(0.0, (double)max_uintx)                                     \
                                                                             \
-  product(bool, G1AllowPreventiveGC, true, DIAGNOSTIC,                       \
+  product(uint, G1RemSetFreeMemoryRescheduleDelayMillis, 10, EXPERIMENTAL,  \
+          "Time after which the card set free memory task reschedules "     \
+          "itself if there is work remaining.")                             \
+          range(1, UINT_MAX)                                                \
+                                                                            \
+  product(double, G1RemSetFreeMemoryStepDurationMillis, 1, EXPERIMENTAL,    \
+          "The amount of time that the free memory task should spend "      \
+          "before a pause of G1RemSetFreeMemoryRescheduleDelayMillis "      \
+          "length.")                                                        \
+          range(1e-3, 1e+6)                                                 \
+                                                                            \
+  product(double, G1RemSetFreeMemoryKeepExcessRatio, 0.1, EXPERIMENTAL,     \
+          "The percentage of free card set memory that G1 should keep as "  \
+          "percentage of the currently used memory.")                       \
+          range(0.0, 1.0)                                                   \
+                                                                            \
+  product(bool, G1AllowPreventiveGC, true, DIAGNOSTIC,                      \
           "Allows collections to be triggered proactively based on the      \
            number of free regions and the expected survival rates in each   \
            section of the heap.")

src/hotspot/share/gc/g1/heapRegion.cpp

@@ -34,7 +34,7 @@
 #include "gc/g1/heapRegion.inline.hpp"
 #include "gc/g1/heapRegionBounds.inline.hpp"
 #include "gc/g1/heapRegionManager.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/g1/heapRegionTracer.hpp"
 #include "gc/shared/genOopClosures.inline.hpp"
 #include "logging/log.hpp"
@@ -226,7 +226,8 @@ void HeapRegion::clear_humongous() {
 
 HeapRegion::HeapRegion(uint hrm_index,
                        G1BlockOffsetTable* bot,
-                       MemRegion mr) :
+                       MemRegion mr,
+                       G1CardSetConfiguration* config) :
   _bottom(mr.start()),
   _end(mr.end()),
   _top(NULL),
@@ -252,7 +253,7 @@ HeapRegion::HeapRegion(uint hrm_index,
   assert(Universe::on_page_boundary(mr.start()) && Universe::on_page_boundary(mr.end()),
          "invalid space boundaries");
 
-  _rem_set = new HeapRegionRemSet(bot, this);
+  _rem_set = new HeapRegionRemSet(this, config);
   initialize();
 }
 
@@ -610,11 +611,12 @@ public:
           if (!_failures) {
             log.error("----------");
           }
+          LogStream ls(log.error());
+          to->rem_set()->print_info(&ls, p);
           log.error("Missing rem set entry:");
           log.error("Field " PTR_FORMAT " of obj " PTR_FORMAT " in region " HR_FORMAT,
                     p2i(p), p2i(_containing_obj), HR_FORMAT_PARAMS(from));
           ResourceMark rm;
-          LogStream ls(log.error());
           _containing_obj->print_on(&ls);
           log.error("points to obj " PTR_FORMAT " in region " HR_FORMAT " remset %s",
                     p2i(obj), HR_FORMAT_PARAMS(to), to->rem_set()->get_state_str());

src/hotspot/share/gc/g1/heapRegion.hpp

@@ -36,6 +36,8 @@
 #include "runtime/mutex.hpp"
 #include "utilities/macros.hpp"
 
+class G1CardSetConfiguration;
+class G1CardSetMemoryManager;
 class G1CollectedHeap;
 class G1CMBitMap;
 class G1Predictions;
@@ -281,7 +283,10 @@ private:
   // starting at p extending to at most the prev TAMS using the given mark bitmap.
   inline size_t block_size_using_bitmap(const HeapWord* p, const G1CMBitMap* const prev_bitmap) const;
 public:
-  HeapRegion(uint hrm_index, G1BlockOffsetTable* bot, MemRegion mr);
+  HeapRegion(uint hrm_index,
+             G1BlockOffsetTable* bot,
+             MemRegion mr,
+             G1CardSetConfiguration* config);
 
   // If this region is a member of a HeapRegionManager, the index in that
   // sequence, otherwise -1.
@@ -445,6 +450,7 @@ public:
   // Unsets the humongous-related fields on the region.
   void clear_humongous();
 
+  void set_rem_set(HeapRegionRemSet* rem_set) { _rem_set = rem_set; }
   // If the region has a remembered set, return a pointer to it.
   HeapRegionRemSet* rem_set() const {
     return _rem_set;

src/hotspot/share/gc/g1/heapRegionRemSet.cpp

@@ -22,13 +22,14 @@
  *
  */
 
+#include <cstdio>
+
 #include "precompiled.hpp"
 #include "gc/g1/g1BlockOffsetTable.inline.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1ConcurrentRefine.hpp"
 #include "gc/g1/heapRegionManager.inline.hpp"
 #include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/sparsePRT.inline.hpp"
 #include "memory/allocation.hpp"
 #include "memory/padded.inline.hpp"
 #include "oops/oop.inline.hpp"
@@ -44,371 +45,19 @@
 const char* HeapRegionRemSet::_state_strings[] =  {"Untracked", "Updating", "Complete"};
 const char* HeapRegionRemSet::_short_state_strings[] =  {"UNTRA", "UPDAT", "CMPLT"};
 
-PerRegionTable* PerRegionTable::alloc(HeapRegion* hr) {
-  PerRegionTable* fl = _free_list;
-  while (fl != NULL) {
-    PerRegionTable* nxt = fl->next();
-    PerRegionTable* res = Atomic::cmpxchg(&_free_list, fl, nxt);
-    if (res == fl) {
-      fl->init(hr, true);
-      return fl;
-    } else {
-      fl = _free_list;
-    }
-  }
-  assert(fl == NULL, "Loop condition.");
-  return new PerRegionTable(hr);
-}
-
-PerRegionTable* volatile PerRegionTable::_free_list = NULL;
-
-size_t OtherRegionsTable::_max_fine_entries = 0;
-size_t OtherRegionsTable::_mod_max_fine_entries_mask = 0;
-size_t OtherRegionsTable::_fine_eviction_stride = 0;
-size_t OtherRegionsTable::_fine_eviction_sample_size = 0;
-
-OtherRegionsTable::OtherRegionsTable(Mutex* m) :
-  _g1h(G1CollectedHeap::heap()),
-  _m(m),
-  _num_occupied(0),
-  _coarse_map(mtGC),
-  _has_coarse_entries(false),
-  _fine_grain_regions(NULL),
-  _n_fine_entries(0),
-  _first_all_fine_prts(NULL),
-  _last_all_fine_prts(NULL),
-  _fine_eviction_start(0),
-  _sparse_table()
-{
-  typedef PerRegionTable* PerRegionTablePtr;
-
-  if (_max_fine_entries == 0) {
-    assert(_mod_max_fine_entries_mask == 0, "Both or none.");
-    size_t max_entries_log = (size_t)log2i(G1RSetRegionEntries);
-    _max_fine_entries = (size_t)1 << max_entries_log;
-    _mod_max_fine_entries_mask = _max_fine_entries - 1;
-
-    assert(_fine_eviction_sample_size == 0
-           && _fine_eviction_stride == 0, "All init at same time.");
-    _fine_eviction_sample_size = MAX2((size_t)4, max_entries_log);
-    _fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size;
-  }
-
-  _fine_grain_regions = NEW_C_HEAP_ARRAY(PerRegionTablePtr, _max_fine_entries, mtGC);
-  for (size_t i = 0; i < _max_fine_entries; i++) {
-    _fine_grain_regions[i] = NULL;
-  }
-}
-
-void OtherRegionsTable::link_to_all(PerRegionTable* prt) {
-  // We always append to the beginning of the list for convenience;
-  // the order of entries in this list does not matter.
-  if (_first_all_fine_prts != NULL) {
-    prt->set_next(_first_all_fine_prts);
-  } else {
-    // this is the first element we insert. Adjust the "last" pointer
-    _last_all_fine_prts = prt;
-    assert(prt->next() == NULL, "just checking");
-  }
-  _first_all_fine_prts = prt;
-
-  assert(_first_all_fine_prts == prt, "just checking");
-  assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) ||
-         (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL),
-         "just checking");
-  assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL,
-         "just checking");
-}
-
-CardIdx_t OtherRegionsTable::card_within_region(OopOrNarrowOopStar within_region, HeapRegion* hr) {
-  assert(hr->is_in_reserved(within_region),
-         "HeapWord " PTR_FORMAT " is outside of region %u [" PTR_FORMAT ", " PTR_FORMAT ")",
-         p2i(within_region), hr->hrm_index(), p2i(hr->bottom()), p2i(hr->end()));
-  CardIdx_t result = (CardIdx_t)(pointer_delta((HeapWord*)within_region, hr->bottom()) >> (CardTable::card_shift - LogHeapWordSize));
-  return result;
-}
-
-void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, uint tid) {
-  // Note that this may be a continued H region.
-  HeapRegion* from_hr = _g1h->heap_region_containing(from);
-  RegionIdx_t from_hrm_ind = (RegionIdx_t) from_hr->hrm_index();
-
-  // If the region is already coarsened, return.
-  if (is_region_coarsened(from_hrm_ind)) {
-    assert(contains_reference(from), "We just found " PTR_FORMAT " in the Coarse table", p2i(from));
-    return;
-  }
-
-  size_t num_added_by_coarsening = 0;
-  // Otherwise find a per-region table to add it to.
-  size_t ind = from_hrm_ind & _mod_max_fine_entries_mask;
-  PerRegionTable* prt = find_region_table(ind, from_hr);
-  if (prt == NULL) {
-    MutexLocker x(_m, Mutex::_no_safepoint_check_flag);
-
-    // Rechecking if the region is coarsened, while holding the lock.
-    if (is_region_coarsened(from_hrm_ind)) {
-      assert(contains_reference_locked(from), "We just found " PTR_FORMAT " in the Coarse table", p2i(from));
-      return;
-    }
-
-    // Confirm that it's really not there...
-    prt = find_region_table(ind, from_hr);
-    if (prt == NULL) {
-
-      CardIdx_t card_index = card_within_region(from, from_hr);
-
-      SparsePRT::AddCardResult result = _sparse_table.add_card(from_hrm_ind, card_index);
-      if (result != SparsePRT::overflow) {
-        if (result == SparsePRT::added) {
-          Atomic::inc(&_num_occupied, memory_order_relaxed);
-        }
-        assert(contains_reference_locked(from), "We just added " PTR_FORMAT " to the Sparse table", p2i(from));
-        return;
-      }
-
-      // Sparse PRT returned overflow (sparse table is full)
-
-      if (_n_fine_entries == _max_fine_entries) {
-        prt = delete_region_table(num_added_by_coarsening);
-        // There is no need to clear the links to the 'all' list here:
-        // prt will be reused immediately, i.e. remain in the 'all' list.
-        prt->init(from_hr, false /* clear_links_to_all_list */);
-      } else {
-        prt = PerRegionTable::alloc(from_hr);
-        link_to_all(prt);
-      }
-
-      PerRegionTable* first_prt = _fine_grain_regions[ind];
-      prt->set_collision_list_next(first_prt);
-      // The assignment into _fine_grain_regions allows the prt to
-      // start being used concurrently. In addition to
-      // collision_list_next which must be visible (else concurrent
-      // parsing of the list, if any, may fail to see other entries),
-      // the content of the prt must be visible (else for instance
-      // some mark bits may not yet seem cleared or a 'later' update
-      // performed by a concurrent thread could be undone when the
-      // zeroing becomes visible). This requires store ordering.
-      Atomic::release_store(&_fine_grain_regions[ind], prt);
-      _n_fine_entries++;
-
-      // Transfer from sparse to fine-grain. The cards from the sparse table
-      // were already added to the total in _num_occupied.
-      SparsePRTEntry *sprt_entry = _sparse_table.get_entry(from_hrm_ind);
-      assert(sprt_entry != NULL, "There should have been an entry");
-      for (int i = 0; i < sprt_entry->num_valid_cards(); i++) {
-        CardIdx_t c = sprt_entry->card(i);
-        prt->add_card(c);
-      }
-      // Now we can delete the sparse entry.
-      bool res = _sparse_table.delete_entry(from_hrm_ind);
-      assert(res, "It should have been there.");
-    }
-    assert(prt != NULL && prt->hr() == from_hr, "consequence");
-  }
-  // Note that we can't assert "prt->hr() == from_hr", because of the
-  // possibility of concurrent reuse.  But see head comment of
-  // OtherRegionsTable for why this is OK.
-  assert(prt != NULL, "Inv");
-
-  if (prt->add_reference(from)) {
-    num_added_by_coarsening++;
-  }
-  Atomic::add(&_num_occupied, num_added_by_coarsening, memory_order_relaxed);
-  assert(contains_reference(from), "We just added " PTR_FORMAT " to the PRT (%d)", p2i(from), prt->contains_reference(from));
-}
-
-PerRegionTable*
-OtherRegionsTable::find_region_table(size_t ind, HeapRegion* hr) const {
-  assert(ind < _max_fine_entries, "Preconditions.");
-  PerRegionTable* prt = _fine_grain_regions[ind];
-  while (prt != NULL && prt->hr() != hr) {
-    prt = prt->collision_list_next();
-  }
-  // Loop postcondition is the method postcondition.
-  return prt;
-}
-
-jint OtherRegionsTable::_n_coarsenings = 0;
-
-PerRegionTable* OtherRegionsTable::delete_region_table(size_t& added_by_deleted) {
-  assert(_m->owned_by_self(), "Precondition");
-  assert(_n_fine_entries == _max_fine_entries, "Precondition");
-  PerRegionTable* max = NULL;
-  jint max_occ = 0;
-  PerRegionTable** max_prev = NULL;
-
-  size_t i = _fine_eviction_start;
-  for (size_t k = 0; k < _fine_eviction_sample_size; k++) {
-    size_t ii = i;
-    // Make sure we get a non-NULL sample.
-    while (_fine_grain_regions[ii] == NULL) {
-      ii++;
-      if (ii == _max_fine_entries) ii = 0;
-      guarantee(ii != i, "We must find one.");
-    }
-    PerRegionTable** prev = &_fine_grain_regions[ii];
-    PerRegionTable* cur = *prev;
-    while (cur != NULL) {
-      jint cur_occ = cur->occupied();
-      if (max == NULL || cur_occ > max_occ) {
-        max = cur;
-        max_prev = prev;
-        max_occ = cur_occ;
-      }
-      prev = cur->collision_list_next_addr();
-      cur = cur->collision_list_next();
-    }
-    i = i + _fine_eviction_stride;
-    if (i >= _n_fine_entries) i = i - _n_fine_entries;
-  }
-
-  _fine_eviction_start++;
-
-  if (_fine_eviction_start >= _n_fine_entries) {
-    _fine_eviction_start -= _n_fine_entries;
-  }
-
-  guarantee(max != NULL, "Since _n_fine_entries > 0");
-  guarantee(max_prev != NULL, "Since max != NULL.");
-
-  // Ensure the corresponding coarse bit is set.
-  size_t max_hrm_index = (size_t) max->hr()->hrm_index();
-  if (Atomic::load(&_has_coarse_entries)) {
-    _coarse_map.at_put(max_hrm_index, true);
-  } else {
-    // This will lazily initialize an uninitialized bitmap
-    _coarse_map.reinitialize(G1CollectedHeap::heap()->max_reserved_regions());
-    assert(!_coarse_map.at(max_hrm_index), "No coarse entries");
-    _coarse_map.at_put(max_hrm_index, true);
-    // Release store guarantees that the bitmap has initialized before any
-    // concurrent reader will ever see _has_coarse_entries is true
-    // (when read with load_acquire)
-    Atomic::release_store(&_has_coarse_entries, true);
-  }
-
-  added_by_deleted = HeapRegion::CardsPerRegion - max_occ;
-  // Unsplice.
-  *max_prev = max->collision_list_next();
-  Atomic::inc(&_n_coarsenings);
-  _n_fine_entries--;
-  return max;
-}
-
-bool OtherRegionsTable::occupancy_less_or_equal_than(size_t limit) const {
-  return occupied() <= limit;
-}
-
-bool OtherRegionsTable::is_empty() const {
-  return occupied() == 0;
-}
-
-size_t OtherRegionsTable::occupied() const {
-  return _num_occupied;
-}
-
-size_t OtherRegionsTable::mem_size() const {
-  size_t sum = 0;
-  // all PRTs are of the same size so it is sufficient to query only one of them.
-  if (_first_all_fine_prts != NULL) {
-    assert(_last_all_fine_prts != NULL &&
-      _first_all_fine_prts->mem_size() == _last_all_fine_prts->mem_size(), "check that mem_size() is constant");
-    sum += _first_all_fine_prts->mem_size() * _n_fine_entries;
-  }
-  sum += (sizeof(PerRegionTable*) * _max_fine_entries);
-  sum += (_coarse_map.size_in_words() * HeapWordSize);
-  sum += (_sparse_table.mem_size());
-  sum += sizeof(OtherRegionsTable) - sizeof(_sparse_table); // Avoid double counting above.
-  return sum;
-}
-
-size_t OtherRegionsTable::static_mem_size() {
-  return G1FromCardCache::static_mem_size();
-}
-
-size_t OtherRegionsTable::fl_mem_size() {
-  return PerRegionTable::fl_mem_size();
-}
-
-void OtherRegionsTable::clear() {
-  // if there are no entries, skip this step
-  if (_first_all_fine_prts != NULL) {
-    guarantee(_first_all_fine_prts != NULL && _last_all_fine_prts != NULL, "just checking");
-    PerRegionTable::bulk_free(_first_all_fine_prts, _last_all_fine_prts);
-    memset(_fine_grain_regions, 0, _max_fine_entries * sizeof(_fine_grain_regions[0]));
-  } else {
-    guarantee(_first_all_fine_prts == NULL && _last_all_fine_prts == NULL, "just checking");
-  }
-
-  _first_all_fine_prts = _last_all_fine_prts = NULL;
-  _sparse_table.clear();
-  if (Atomic::load(&_has_coarse_entries)) {
-    _coarse_map.clear();
-  }
-  _n_fine_entries = 0;
-  Atomic::store(&_has_coarse_entries, false);
-
-  _num_occupied = 0;
-}
-
-bool OtherRegionsTable::contains_reference(OopOrNarrowOopStar from) const {
-  // Cast away const in this case.
-  MutexLocker x((Mutex*)_m, Mutex::_no_safepoint_check_flag);
-  return contains_reference_locked(from);
-}
-
-bool OtherRegionsTable::contains_reference_locked(OopOrNarrowOopStar from) const {
-  HeapRegion* hr = _g1h->heap_region_containing(from);
-  RegionIdx_t hr_ind = (RegionIdx_t) hr->hrm_index();
-  // Is this region in the coarse map?
-  if (is_region_coarsened(hr_ind)) return true;
-
-  PerRegionTable* prt = find_region_table(hr_ind & _mod_max_fine_entries_mask,
-                                          hr);
-  if (prt != NULL) {
-    return prt->contains_reference(from);
-  } else {
-    CardIdx_t card_index = card_within_region(from, hr);
-    return _sparse_table.contains_card(hr_ind, card_index);
-  }
-}
-
-// A load_acquire on _has_coarse_entries - coupled with the release_store in
-// delete_region_table - guarantees we don't access _coarse_map before
-// it's been properly initialized.
-bool OtherRegionsTable::is_region_coarsened(RegionIdx_t from_hrm_ind) const {
-  return Atomic::load_acquire(&_has_coarse_entries) && _coarse_map.at(from_hrm_ind);
-}
-
-HeapRegionRemSet::HeapRegionRemSet(G1BlockOffsetTable* bot,
-                                   HeapRegion* hr)
-  : _bot(bot),
-    _code_roots(),
-    _m(Mutex::leaf, FormatBuffer<128>("HeapRegionRemSet lock #%u", hr->hrm_index()), true, Mutex::_safepoint_check_never),
-    _other_regions(&_m),
-    _hr(hr),
-    _state(Untracked)
-{
-}
+HeapRegionRemSet::HeapRegionRemSet(HeapRegion* hr,
+                                   G1CardSetConfiguration* config) :
+  _m(Mutex::leaf + 1, FormatBuffer<128>("HeapRegionRemSet lock #%u", hr->hrm_index()), true, Monitor::_safepoint_check_never),
+  _code_roots(),
+  _card_set_mm(config, G1CardSetFreePool::free_list_pool()),
+  _card_set(config, &_card_set_mm),
+  _hr(hr),
+  _state(Untracked) { }
 
 void HeapRegionRemSet::clear_fcc() {
   G1FromCardCache::clear(_hr->hrm_index());
 }
 
-void HeapRegionRemSet::setup_remset_size() {
-  const int LOG_M = 20;
-  guarantee(HeapRegion::LogOfHRGrainBytes >= LOG_M, "Code assumes the region size >= 1M, but is " SIZE_FORMAT "B", HeapRegion::GrainBytes);
-
-  int region_size_log_mb = HeapRegion::LogOfHRGrainBytes - LOG_M;
-  if (FLAG_IS_DEFAULT(G1RSetSparseRegionEntries)) {
-    G1RSetSparseRegionEntries = G1RSetSparseRegionEntriesBase * ((size_t)1 << (region_size_log_mb + 1));
-  }
-  if (FLAG_IS_DEFAULT(G1RSetRegionEntries)) {
-    G1RSetRegionEntries = G1RSetRegionEntriesBase * (region_size_log_mb + 1);
-  }
-  guarantee(G1RSetSparseRegionEntries > 0 && G1RSetRegionEntries > 0 , "Sanity");
-}
-
 void HeapRegionRemSet::clear(bool only_cardset) {
   MutexLocker x(&_m, Mutex::_no_safepoint_check_flag);
   clear_locked(only_cardset);
@@ -419,11 +68,15 @@ void HeapRegionRemSet::clear_locked(bool only_cardset) {
     _code_roots.clear();
   }
   clear_fcc();
-  _other_regions.clear();
+  _card_set.clear();
   set_state_empty();
   assert(occupied() == 0, "Should be clear.");
 }
 
+void HeapRegionRemSet::print_static_mem_size(outputStream* out) {
+  out->print_cr("  Static structures = " SIZE_FORMAT, HeapRegionRemSet::static_mem_size());
+}
+
 // Code roots support
 //
 // The code root set is protected by two separate locking schemes

src/hotspot/share/gc/g1/heapRegionRemSet.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,299 +25,62 @@
 #ifndef SHARE_GC_G1_HEAPREGIONREMSET_HPP
 #define SHARE_GC_G1_HEAPREGIONREMSET_HPP
 
+#include "gc/g1/g1CardSet.hpp"
+#include "gc/g1/g1CardSetMemory.hpp"
 #include "gc/g1/g1CodeCacheRemSet.hpp"
 #include "gc/g1/g1FromCardCache.hpp"
-#include "gc/g1/sparsePRT.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/safepoint.hpp"
 #include "utilities/bitMap.hpp"
 
-// Remembered set for a heap region.  Represent a set of "cards" that
-// contain pointers into the owner heap region.  Cards are defined somewhat
-// abstractly, in terms of what the "BlockOffsetTable" in use can parse.
-
-class G1CollectedHeap;
-class G1BlockOffsetTable;
-class G1CardLiveData;
-class HeapRegion;
-class PerRegionTable;
-class SparsePRT;
-class nmethod;
-
-// The "_coarse_map" is a bitmap with one bit for each region, where set
-// bits indicate that the corresponding region may contain some pointer
-// into the owning region.
-
-// The "_fine_grain_entries" array is an open hash table of PerRegionTables
-// (PRTs), indicating regions for which we're keeping the RS as a set of
-// cards.  The strategy is to cap the size of the fine-grain table,
-// deleting an entry and setting the corresponding coarse-grained bit when
-// we would overflow this cap.
-
-// We use a mixture of locking and lock-free techniques here.  We allow
-// threads to locate PRTs without locking, but threads attempting to alter
-// a bucket list obtain a lock.  This means that any failing attempt to
-// find a PRT must be retried with the lock.  It might seem dangerous that
-// a read can find a PRT that is concurrently deleted.  This is all right,
-// because:
-//
-//   1) We only actually free PRT's at safe points (though we reuse them at
-//      other times).
-//   2) We find PRT's in an attempt to add entries.  If a PRT is deleted,
-//      it's _coarse_map bit is set, so the that we were attempting to add
-//      is represented.  If a deleted PRT is re-used, a thread adding a bit,
-//      thinking the PRT is for a different region, does no harm.
-
-class OtherRegionsTable {
-  G1CollectedHeap* _g1h;
-  Mutex*           _m;
-
-  size_t volatile _num_occupied;
-
-  // These are protected by "_m".
-  CHeapBitMap   _coarse_map;
-  bool volatile _has_coarse_entries;
-  static jint   _n_coarsenings;
-
-  PerRegionTable** _fine_grain_regions;
-  size_t           _n_fine_entries;
-
-  // The fine grain remembered sets are linked together using
-  // their 'next' fields.
-  // This allows fast bulk freeing of all the fine grain remembered
-  // set entries, and fast finding of all of them without iterating
-  // over the _fine_grain_regions table.
-  PerRegionTable * _first_all_fine_prts;
-  PerRegionTable * _last_all_fine_prts;
-
-  // Used to sample a subset of the fine grain PRTs to determine which
-  // PRT to evict and coarsen.
-  size_t        _fine_eviction_start;
-  static size_t _fine_eviction_stride;
-  static size_t _fine_eviction_sample_size;
-
-  SparsePRT   _sparse_table;
-
-  // These are static after init.
-  static size_t _max_fine_entries;
-  static size_t _mod_max_fine_entries_mask;
-
-  // Requires "prt" to be the first element of the bucket list appropriate
-  // for "hr".  If this list contains an entry for "hr", return it,
-  // otherwise return "NULL".
-  PerRegionTable* find_region_table(size_t ind, HeapRegion* hr) const;
-
-  // Find, delete, and return a candidate PerRegionTable, if any exists,
-  // adding the deleted region to the coarse bitmap.  Requires the caller
-  // to hold _m, and the fine-grain table to be full.
-  PerRegionTable* delete_region_table(size_t& added_by_deleted);
-
-  // link/add the given fine grain remembered set into the "all" list
-  void link_to_all(PerRegionTable * prt);
-
-  bool contains_reference_locked(OopOrNarrowOopStar from) const;
-
-public:
-  // Create a new remembered set. The given mutex is used to ensure consistency.
-  OtherRegionsTable(Mutex* m);
-
-  template <class Closure>
-  void iterate(Closure& v);
-
-  // Returns the card index of the given within_region pointer relative to the bottom
-  // of the given heap region.
-  static CardIdx_t card_within_region(OopOrNarrowOopStar within_region, HeapRegion* hr);
-  // Adds the reference from "from to this remembered set.
-  void add_reference(OopOrNarrowOopStar from, uint tid);
-
-  // Returns whether the remembered set contains the given reference.
-  bool contains_reference(OopOrNarrowOopStar from) const;
-
-  // Returns whether this remembered set (and all sub-sets) have an occupancy
-  // that is less or equal than the given occupancy.
-  bool occupancy_less_or_equal_than(size_t limit) const;
-
-  // Returns whether this remembered set (and all sub-sets) does not contain any entry.
-  bool is_empty() const;
-
-  // Returns the number of cards contained in this remembered set.
-  size_t occupied() const;
-
-  static jint n_coarsenings() { return _n_coarsenings; }
-
-  // Returns size of the actual remembered set containers in bytes.
-  size_t mem_size() const;
-  // Returns the size of static data in bytes.
-  static size_t static_mem_size();
-  // Returns the size of the free list content in bytes.
-  static size_t fl_mem_size();
-
-  // Clear the entire contents of this remembered set.
-  void clear();
-
-  // Safe for use by concurrent readers outside _m
-  bool is_region_coarsened(RegionIdx_t from_hrm_ind) const;
-};
-
-class PerRegionTable: public CHeapObj<mtGC> {
-  friend class OtherRegionsTable;
-
-  HeapRegion*     _hr;
-  CHeapBitMap     _bm;
-  jint            _occupied;
-
-  // next pointer for free/allocated 'all' list
-  PerRegionTable* _next;
-
-  // next pointer in collision list
-  PerRegionTable * _collision_list_next;
-
-  // Global free list of PRTs
-  static PerRegionTable* volatile _free_list;
-
-protected:
-  PerRegionTable(HeapRegion* hr) :
-    _hr(hr),
-    _bm(HeapRegion::CardsPerRegion, mtGC),
-    _occupied(0),
-    _next(NULL),
-    _collision_list_next(NULL)
-  {}
-
-public:
-  // We need access in order to union things into the base table.
-  BitMap* bm() { return &_bm; }
-
-  HeapRegion* hr() const { return Atomic::load_acquire(&_hr); }
-
-  jint occupied() const {
-    return _occupied;
-  }
-
-  void init(HeapRegion* hr, bool clear_links_to_all_list);
-
-  inline bool add_reference(OopOrNarrowOopStar from);
-
-  inline bool add_card(CardIdx_t from_card_index);
-
-  // (Destructively) union the bitmap of the current table into the given
-  // bitmap (which is assumed to be of the same size.)
-  void union_bitmap_into(BitMap* bm) {
-    bm->set_union(_bm);
-  }
-
-  // Mem size in bytes.
-  size_t mem_size() const {
-    return sizeof(PerRegionTable) + _bm.size_in_words() * HeapWordSize;
-  }
-
-  // Requires "from" to be in "hr()".
-  bool contains_reference(OopOrNarrowOopStar from) const {
-    assert(hr()->is_in_reserved(from), "Precondition.");
-    size_t card_ind = pointer_delta(from, hr()->bottom(),
-                                    G1CardTable::card_size);
-    return _bm.at(card_ind);
-  }
-
-  // Bulk-free the PRTs from prt to last, assumes that they are
-  // linked together using their _next field.
-  static void bulk_free(PerRegionTable* prt, PerRegionTable* last) {
-    while (true) {
-      PerRegionTable* fl = _free_list;
-      last->set_next(fl);
-      PerRegionTable* res = Atomic::cmpxchg(&_free_list, fl, prt);
-      if (res == fl) {
-        return;
-      }
-    }
-    ShouldNotReachHere();
-  }
-
-  static void free(PerRegionTable* prt) {
-    bulk_free(prt, prt);
-  }
-
-  // Returns an initialized PerRegionTable instance.
-  static PerRegionTable* alloc(HeapRegion* hr);
-
-  PerRegionTable* next() const { return _next; }
-  void set_next(PerRegionTable* next) { _next = next; }
-
-  // Accessor and Modification routines for the pointer for the
-  // singly linked collision list that links the PRTs within the
-  // OtherRegionsTable::_fine_grain_regions hash table.
-  //
-
-  PerRegionTable* collision_list_next() const {
-    return _collision_list_next;
-  }
-
-  void set_collision_list_next(PerRegionTable* next) {
-    _collision_list_next = next;
-  }
-
-  PerRegionTable** collision_list_next_addr() {
-    return &_collision_list_next;
-  }
-
-  static size_t fl_mem_size() {
-    PerRegionTable* cur = _free_list;
-    size_t res = 0;
-    while (cur != NULL) {
-      res += cur->mem_size();
-      cur = cur->next();
-    }
-    return res;
-  }
-
-  static void test_fl_mem_size();
-};
+class outputStream;
 
 class HeapRegionRemSet : public CHeapObj<mtGC> {
   friend class VMStructs;
 
-private:
-  G1BlockOffsetTable* _bot;
-
+  Mutex _m;
   // A set of code blobs (nmethods) whose code contains pointers into
   // the region that owns this RSet.
   G1CodeRootSet _code_roots;
 
-  Mutex _m;
+  G1CardSetMemoryManager _card_set_mm;
 
-  OtherRegionsTable _other_regions;
+  // The set of cards in the Java heap
+  G1CardSet _card_set;
 
   HeapRegion* _hr;
 
+  inline void split_card(OopOrNarrowOopStar from, uint& card_region, uint& card_within_region) const;
   void clear_fcc();
 
 public:
-  HeapRegionRemSet(G1BlockOffsetTable* bot, HeapRegion* hr);
+  HeapRegionRemSet(HeapRegion* hr, G1CardSetConfiguration* config);
 
   // Setup sparse and fine-grain tables sizes.
-  static void setup_remset_size();
+  bool cardset_is_empty() const {
+    return _card_set.is_empty();
+  }
 
   bool is_empty() const {
-    return (strong_code_roots_list_length() == 0) && _other_regions.is_empty();
+    return (strong_code_roots_list_length() == 0) && cardset_is_empty();
   }
 
   bool occupancy_less_or_equal_than(size_t occ) const {
-    return (strong_code_roots_list_length() == 0) && _other_regions.occupancy_less_or_equal_than(occ);
+    return (strong_code_roots_list_length() == 0) && _card_set.occupancy_less_or_equal_to(occ);
   }
 
-  // For each PRT in the card (remembered) set call one of the following methods
-  // of the given closure:
-  //
-  // set_full_region_dirty(uint region_idx) - pass the region index for coarse PRTs
-  // set_bitmap_dirty(uint region_idx, BitMap* bitmap) - pass the region index and bitmap for fine PRTs
-  // set_cards_dirty(uint region_idx, elem_t* cards, uint num_cards) - pass region index and cards for sparse PRTs
-  template <class Closure>
-  inline void iterate_prts(Closure& cl);
+  // Iterate the card based remembered set for merging them into the card table.
+  // The passed closure must be a CardOrRangeVisitor; we use a template parameter
+  // to pass it in to facilitate inlining as much as possible.
+  template <class CardOrRangeVisitor>
+  inline void iterate_for_merge(CardOrRangeVisitor& cl);
 
   size_t occupied() {
-    return _other_regions.occupied();
+    return _card_set.occupied();
   }
 
-  static jint n_coarsenings() { return OtherRegionsTable::n_coarsenings(); }
+  // Coarsening statistics since VM start.
+  static G1CardSetCoarsenStats coarsen_stats() { return G1CardSet::coarsen_stats(); }
 
 private:
   enum RemSetState {
@@ -339,74 +102,42 @@ public:
   bool is_updating() { return _state == Updating; }
   bool is_complete() { return _state == Complete; }
 
-  void set_state_empty() {
-    guarantee(SafepointSynchronize::is_at_safepoint() || !is_tracked(), "Should only set to Untracked during safepoint but is %s.", get_state_str());
-    if (_state == Untracked) {
-      return;
-    }
-    clear_fcc();
-    _state = Untracked;
-  }
+  inline void set_state_empty();
+  inline void set_state_updating();
+  inline void set_state_complete();
 
-  void set_state_updating() {
-    guarantee(SafepointSynchronize::is_at_safepoint() && !is_tracked(), "Should only set to Updating from Untracked during safepoint but is %s", get_state_str());
-    clear_fcc();
-    _state = Updating;
-  }
-
-  void set_state_complete() {
-    clear_fcc();
-    _state = Complete;
-  }
-
-  void add_reference(OopOrNarrowOopStar from, uint tid) {
-    RemSetState state = _state;
-    if (state == Untracked) {
-      return;
-    }
-
-    uint cur_idx = _hr->hrm_index();
-    uintptr_t from_card = uintptr_t(from) >> CardTable::card_shift;
-
-    if (G1FromCardCache::contains_or_replace(tid, cur_idx, from_card)) {
-      assert(contains_reference(from), "We just found " PTR_FORMAT " in the FromCardCache", p2i(from));
-      return;
-    }
-
-    _other_regions.add_reference(from, tid);
-  }
+  inline void add_reference(OopOrNarrowOopStar from, uint tid);
 
   // The region is being reclaimed; clear its remset, and any mention of
   // entries for this region in other remsets.
   void clear(bool only_cardset = false);
   void clear_locked(bool only_cardset = false);
 
-  // The actual # of bytes this hr_remset takes up.
-  // Note also includes the strong code root set.
+  G1CardSetMemoryStats card_set_memory_stats() const { return _card_set_mm.memory_stats(); }
+
+  // The actual # of bytes this hr_remset takes up. Also includes the strong code
+  // root set.
   size_t mem_size() {
-    MutexLocker x(&_m, Mutex::_no_safepoint_check_flag);
-    return _other_regions.mem_size()
-      // This correction is necessary because the above includes the second
-      // part.
-      + (sizeof(HeapRegionRemSet) - sizeof(OtherRegionsTable))
+    return _card_set.mem_size()
+      + (sizeof(HeapRegionRemSet) - sizeof(G1CardSet)) // Avoid double-counting G1CardSet.
       + strong_code_roots_mem_size();
   }
 
+  size_t wasted_mem_size() {
+    return _card_set.wasted_mem_size();
+  }
+
   // Returns the memory occupancy of all static data structures associated
   // with remembered sets.
   static size_t static_mem_size() {
-    return OtherRegionsTable::static_mem_size() + G1CodeRootSet::static_mem_size();
+    return G1CardSet::static_mem_size() + G1CodeRootSet::static_mem_size() + sizeof(G1CardSetFreePool);
   }
 
-  // Returns the memory occupancy of all free_list data structures associated
-  // with remembered sets.
-  static size_t fl_mem_size() {
-    return OtherRegionsTable::fl_mem_size();
-  }
+  static void print_static_mem_size(outputStream* out);
 
-  bool contains_reference(OopOrNarrowOopStar from) const {
-    return _other_regions.contains_reference(from);
-  }
+  inline bool contains_reference(OopOrNarrowOopStar from);
+
+  inline void print_info(outputStream* st, OopOrNarrowOopStar from);
 
   // Routines for managing the list of code roots that point into
   // the heap region that owns this RSet.

src/hotspot/share/gc/g1/heapRegionRemSet.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,74 +27,86 @@
 
 #include "gc/g1/heapRegionRemSet.hpp"
 
+#include "gc/g1/g1CardSet.inline.hpp"
+#include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/sparsePRT.hpp"
 #include "runtime/atomic.hpp"
 #include "utilities/bitMap.inline.hpp"
 
-template <class Closure>
-inline void HeapRegionRemSet::iterate_prts(Closure& cl) {
-  _other_regions.iterate(cl);
+void HeapRegionRemSet::set_state_empty() {
+  guarantee(SafepointSynchronize::is_at_safepoint() || !is_tracked(),
+            "Should only set to Untracked during safepoint but is %s.", get_state_str());
+  if (_state == Untracked) {
+    return;
+  }
+  clear_fcc();
+  _state = Untracked;
 }
 
-inline bool PerRegionTable::add_card(CardIdx_t from_card_index) {
-  if (_bm.par_set_bit(from_card_index)) {
-    Atomic::inc(&_occupied, memory_order_relaxed);
-    return true;
-  }
-  return false;
+void HeapRegionRemSet::set_state_updating() {
+  guarantee(SafepointSynchronize::is_at_safepoint() && !is_tracked(),
+            "Should only set to Updating from Untracked during safepoint but is %s", get_state_str());
+  clear_fcc();
+  _state = Updating;
 }
 
-inline bool PerRegionTable::add_reference(OopOrNarrowOopStar from) {
-  // Must make this robust in case "from" is not in "_hr", because of
-  // concurrency.
-
-  HeapRegion* loc_hr = hr();
-  // If the test below fails, then this table was reused concurrently
-  // with this operation.  This is OK, since the old table was coarsened,
-  // and adding a bit to the new table is never incorrect.
-  if (loc_hr->is_in_reserved(from)) {
-    CardIdx_t from_card = OtherRegionsTable::card_within_region(from, loc_hr);
-    return add_card(from_card);
-  }
-  return false;
+void HeapRegionRemSet::set_state_complete() {
+  clear_fcc();
+  _state = Complete;
 }
 
-inline void PerRegionTable::init(HeapRegion* hr, bool clear_links_to_all_list) {
-  if (clear_links_to_all_list) {
-    set_next(NULL);
-  }
-  _collision_list_next = NULL;
-  _occupied = 0;
-  _bm.clear();
-  // Make sure that the bitmap clearing above has been finished before publishing
-  // this PRT to concurrent threads.
-  Atomic::release_store(&_hr, hr);
+
+template <class CardOrRangeVisitor>
+inline void HeapRegionRemSet::iterate_for_merge(CardOrRangeVisitor& cl) {
+  _card_set.iterate_for_merge(cl);
 }
 
-template <class Closure>
-void OtherRegionsTable::iterate(Closure& cl) {
-  if (Atomic::load(&_has_coarse_entries)) {
-    BitMap::idx_t cur = _coarse_map.get_next_one_offset(0);
-    while (cur != _coarse_map.size()) {
-      cl.next_coarse_prt((uint)cur);
-      cur = _coarse_map.get_next_one_offset(cur + 1);
-    }
+void HeapRegionRemSet::split_card(OopOrNarrowOopStar from, uint& card_region, uint& card_within_region) const {
+  HeapRegion* hr = G1CollectedHeap::heap()->heap_region_containing(from);
+  card_region = hr->hrm_index();
+  card_within_region = (uint)(pointer_delta((HeapWord*)from, hr->bottom()) >> (CardTable::card_shift - LogHeapWordSize));
+}
+
+void HeapRegionRemSet::add_reference(OopOrNarrowOopStar from, uint tid) {
+  RemSetState state = _state;
+  if (state == Untracked) {
+    return;
   }
-  {
-    PerRegionTable* cur = _first_all_fine_prts;
-    while (cur != NULL) {
-      cl.next_fine_prt(cur->hr()->hrm_index(), cur->bm());
-      cur = cur->next();
-    }
-  }
-  {
-    SparsePRTBucketIter iter(&_sparse_table);
-    SparsePRTEntry* cur;
-    while (iter.has_next(cur)) {
-      cl.next_sparse_prt(cur->r_ind(), cur->cards(), cur->num_valid_cards());
-    }
+
+  uint cur_idx = _hr->hrm_index();
+  uintptr_t from_card = uintptr_t(from) >> CardTable::card_shift;
+
+  if (G1FromCardCache::contains_or_replace(tid, cur_idx, from_card)) {
+    // We can't check whether the card is in the remembered set - the card container
+    // may be coarsened just now.
+    //assert(contains_reference(from), "We just found " PTR_FORMAT " in the FromCardCache", p2i(from));
+   return;
   }
+
+  uint card_region;
+  uint card_within_region;
+
+  split_card(from, card_region, card_within_region);
+
+  _card_set.add_card(card_region, card_within_region);
+}
+
+bool HeapRegionRemSet::contains_reference(OopOrNarrowOopStar from) {
+  uint card_region;
+  uint card_within_region;
+
+  split_card(from, card_region, card_within_region);
+
+  return _card_set.contains_card(card_region, card_within_region);
+}
+
+void HeapRegionRemSet::print_info(outputStream* st, OopOrNarrowOopStar from) {
+  uint card_region;
+  uint card_within_region;
+
+  split_card(from, card_region, card_within_region);
+
+  _card_set.print_info(st, card_region, card_within_region);
 }
 
 #endif // SHARE_VM_GC_G1_HEAPREGIONREMSET_INLINE_HPP

src/hotspot/share/gc/g1/heapRegionSet.cpp

@@ -25,7 +25,7 @@
 #include "precompiled.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1NUMA.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/g1/heapRegionSet.inline.hpp"
 
 uint FreeRegionList::_unrealistically_long_length = 0;

src/hotspot/share/gc/g1/jvmFlagConstraintsG1.cpp

@@ -28,15 +28,15 @@
 #include "runtime/globals_extension.hpp"
 #include "utilities/globalDefinitions.hpp"
 
-JVMFlag::Error G1RSetRegionEntriesConstraintFunc(intx value, bool verbose) {
+JVMFlag::Error G1RemSetArrayOfCardsEntriesConstraintFunc(uint value, bool verbose) {
   if (!UseG1GC) return JVMFlag::SUCCESS;
 
-  // Default value of G1RSetRegionEntries=0 means will be set ergonomically.
+  // Default value of G1RemSetArrayOfCardsEntries=0 means will be set ergonomically.
   // Minimum value is 1.
-  if (FLAG_IS_CMDLINE(G1RSetRegionEntries) && (value < 1)) {
+  if (FLAG_IS_CMDLINE(G1RemSetArrayOfCardsEntries) && (value < 1)) {
     JVMFlag::printError(verbose,
-                        "G1RSetRegionEntries (" INTX_FORMAT ") must be "
-                        "greater than or equal to 1\n",
+                        "G1RemSetArrayOfCardsEntries (%u) must be "
+                        "greater than or equal to 1.\n",
                         value);
     return JVMFlag::VIOLATES_CONSTRAINT;
   } else {
@@ -44,20 +44,35 @@ JVMFlag::Error G1RSetRegionEntriesConstraintFunc(intx value, bool verbose) {
   }
 }
 
-JVMFlag::Error G1RSetSparseRegionEntriesConstraintFunc(intx value, bool verbose) {
+JVMFlag::Error G1RemSetHowlNumBucketsConstraintFunc(uint value, bool verbose) {
   if (!UseG1GC) return JVMFlag::SUCCESS;
 
-  // Default value of G1RSetSparseRegionEntries=0 means will be set ergonomically.
-  // Minimum value is 1.
-  if (FLAG_IS_CMDLINE(G1RSetSparseRegionEntries) && (value < 1)) {
-    JVMFlag::printError(verbose,
-                        "G1RSetSparseRegionEntries (" INTX_FORMAT ") must be "
-                        "greater than or equal to 1\n",
-                        value);
-    return JVMFlag::VIOLATES_CONSTRAINT;
-  } else {
+  if (!FLAG_IS_CMDLINE(G1RemSetHowlNumBuckets)) {
     return JVMFlag::SUCCESS;
   }
+  if (value == 0 || !is_power_of_2(G1RemSetHowlNumBuckets)) {
+    JVMFlag::printError(verbose,
+                        "G1RemSetHowlNumBuckets (%u) must be a power of two "
+                        "and greater than or equal to 1.\n",
+                        value);
+    return JVMFlag::VIOLATES_CONSTRAINT;
+  }
+  return JVMFlag::SUCCESS;
+}
+
+JVMFlag::Error G1RemSetHowlMaxNumBucketsConstraintFunc(uint value, bool verbose) {
+  if (!UseG1GC) return JVMFlag::SUCCESS;
+
+  if (!FLAG_IS_CMDLINE(G1RemSetHowlMaxNumBuckets)) {
+    return JVMFlag::SUCCESS;
+  }
+  if (!is_power_of_2(G1RemSetHowlMaxNumBuckets)) {
+    JVMFlag::printError(verbose,
+                        "G1RemSetMaxHowlNumBuckets (%u) must be a power of two.\n",
+                        value);
+    return JVMFlag::VIOLATES_CONSTRAINT;
+  }
+  return JVMFlag::SUCCESS;
 }
 
 JVMFlag::Error G1HeapRegionSizeConstraintFunc(size_t value, bool verbose) {

src/hotspot/share/gc/g1/jvmFlagConstraintsG1.hpp

@@ -30,9 +30,12 @@
 
 #define G1_GC_CONSTRAINTS(f)                          \
                                                       \
-  /* G1 Flag Constraints */                           \
-  f(intx,   G1RSetRegionEntriesConstraintFunc)        \
-  f(intx,   G1RSetSparseRegionEntriesConstraintFunc)  \
+  /* G1 Remembered Sets Constraints */                \
+  f(uint,   G1RemSetArrayOfCardsEntriesConstraintFunc)\
+  f(uint,   G1RemSetHowlMaxNumBucketsConstraintFunc)  \
+  f(uint,   G1RemSetHowlNumBucketsConstraintFunc)     \
+                                                      \
+  /* G1 Heap Size Constraints */                      \
   f(size_t, G1HeapRegionSizeConstraintFunc)           \
   f(uintx,  G1NewSizePercentConstraintFunc)           \
   f(uintx,  G1MaxNewSizePercentConstraintFunc)        \

src/hotspot/share/gc/g1/sparsePRT.cpp

@@ -1,311 +0,0 @@
-/*
- * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionBounds.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
-#include "gc/g1/sparsePRT.hpp"
-#include "gc/shared/cardTableBarrierSet.hpp"
-#include "gc/shared/space.inline.hpp"
-#include "memory/allocation.inline.hpp"
-
-// Check that the size of the SparsePRTEntry is evenly divisible by the maximum
-// member type to avoid SIGBUS when accessing them.
-STATIC_ASSERT(sizeof(SparsePRTEntry) % sizeof(int) == 0);
-
-void SparsePRTEntry::init(RegionIdx_t region_ind) {
-  // Check that the card array element type can represent all cards in the region.
-  // Choose a large SparsePRTEntry::card_elem_t (e.g. CardIdx_t) if required.
-  assert(((size_t)1 << (sizeof(SparsePRTEntry::card_elem_t) * BitsPerByte)) *
-         G1CardTable::card_size >= HeapRegionBounds::max_size(), "precondition");
-  assert(G1RSetSparseRegionEntries > 0, "precondition");
-  _region_ind = region_ind;
-  _next_index = RSHashTable::NullEntry;
-  _next_null = 0;
-}
-
-bool SparsePRTEntry::contains_card(CardIdx_t card_index) const {
-  for (int i = 0; i < num_valid_cards(); i++) {
-    if (card(i) == card_index) {
-      return true;
-    }
-  }
-  return false;
-}
-
-SparsePRT::AddCardResult SparsePRTEntry::add_card(CardIdx_t card_index) {
-  for (int i = 0; i < num_valid_cards(); i++) {
-    if (card(i) == card_index) {
-      return SparsePRT::found;
-    }
-  }
-  if (num_valid_cards() < cards_num() - 1) {
-    _cards[_next_null] = (card_elem_t)card_index;
-    _next_null++;
-    return SparsePRT::added;
-   }
-  // Otherwise, we're full.
-  return SparsePRT::overflow;
-}
-
-void SparsePRTEntry::copy_cards(card_elem_t* cards) const {
-  memcpy(cards, _cards, cards_num() * sizeof(card_elem_t));
-}
-
-void SparsePRTEntry::copy_cards(SparsePRTEntry* e) const {
-  copy_cards(e->_cards);
-  assert(_next_null >= 0, "invariant");
-  assert(_next_null <= cards_num(), "invariant");
-  e->_next_null = _next_null;
-}
-
-// ----------------------------------------------------------------------
-
-float RSHashTable::TableOccupancyFactor = 0.5f;
-
-// The empty table can't hold any entries and is effectively immutable
-// This means it can be used as an initial sentinel value
-static int empty_buckets[] = { RSHashTable::NullEntry };
-RSHashTable RSHashTable::empty_table;
-
-RSHashTable::RSHashTable() :
-  _num_entries(0),
-  _capacity(0),
-  _capacity_mask(0),
-  _occupied_entries(0),
-  _entries(NULL),
-  _buckets(empty_buckets),
-  _free_region(0),
-  _free_list(NullEntry) { }
-
-RSHashTable::RSHashTable(size_t capacity) :
-  _num_entries((capacity * TableOccupancyFactor) + 1),
-  _capacity(capacity),
-  _capacity_mask(capacity - 1),
-  _occupied_entries(0),
-  _entries((SparsePRTEntry*)NEW_C_HEAP_ARRAY(char, _num_entries * SparsePRTEntry::size(), mtGC)),
-  _buckets(NEW_C_HEAP_ARRAY(int, capacity, mtGC)),
-  _free_region(0),
-  _free_list(NullEntry)
-{
-  clear();
-}
-
-RSHashTable::~RSHashTable() {
-  // Nothing to free for empty RSHashTable
-  if (_buckets != empty_buckets) {
-    assert(_entries != NULL, "invariant");
-    FREE_C_HEAP_ARRAY(SparsePRTEntry, _entries);
-    FREE_C_HEAP_ARRAY(int, _buckets);
-  }
-}
-
-void RSHashTable::clear() {
-  assert(_buckets != empty_buckets, "Shouldn't call this for the empty_table");
-  _occupied_entries = 0;
-  guarantee(_entries != NULL, "invariant");
-  guarantee(_buckets != NULL, "invariant");
-
-  guarantee(_capacity <= ((size_t)1 << (sizeof(int)*BitsPerByte-1)) - 1,
-                "_capacity too large");
-
-  // This will put -1 == NullEntry in the key field of all entries.
-  memset((void*)_entries, NullEntry, _num_entries * SparsePRTEntry::size());
-  memset((void*)_buckets, NullEntry, _capacity * sizeof(int));
-  _free_list = NullEntry;
-  _free_region = 0;
-}
-
-SparsePRT::AddCardResult RSHashTable::add_card(RegionIdx_t region_ind, CardIdx_t card_index) {
-  assert(this != &empty_table, "can't add a card to the empty table");
-  SparsePRTEntry* e = entry_for_region_ind_create(region_ind);
-  assert(e != NULL && e->r_ind() == region_ind,
-         "Postcondition of call above.");
-  SparsePRT::AddCardResult res = e->add_card(card_index);
-  assert(e->num_valid_cards() > 0, "Postcondition");
-  return res;
-}
-
-SparsePRTEntry* RSHashTable::get_entry(RegionIdx_t region_ind) const {
-  int ind = (int) (region_ind & capacity_mask());
-  int cur_ind = _buckets[ind];
-  SparsePRTEntry* cur;
-  while (cur_ind != NullEntry &&
-         (cur = entry(cur_ind))->r_ind() != region_ind) {
-    cur_ind = cur->next_index();
-  }
-
-  if (cur_ind == NullEntry) return NULL;
-  // Otherwise...
-  assert(cur->r_ind() == region_ind, "Postcondition of loop + test above.");
-  assert(cur->num_valid_cards() > 0, "Inv");
-  return cur;
-}
-
-bool RSHashTable::delete_entry(RegionIdx_t region_ind) {
-  int ind = (int) (region_ind & capacity_mask());
-  int* prev_loc = &_buckets[ind];
-  int cur_ind = *prev_loc;
-  SparsePRTEntry* cur;
-  while (cur_ind != NullEntry &&
-         (cur = entry(cur_ind))->r_ind() != region_ind) {
-    prev_loc = cur->next_index_addr();
-    cur_ind = *prev_loc;
-  }
-
-  if (cur_ind == NullEntry) return false;
-  // Otherwise, splice out "cur".
-  *prev_loc = cur->next_index();
-  free_entry(cur_ind);
-  _occupied_entries--;
-  return true;
-}
-
-SparsePRTEntry*
-RSHashTable::entry_for_region_ind_create(RegionIdx_t region_ind) {
-  SparsePRTEntry* res = get_entry(region_ind);
-  if (res == NULL) {
-    int new_ind = alloc_entry();
-    res = entry(new_ind);
-    res->init(region_ind);
-    // Insert at front.
-    int ind = (int) (region_ind & capacity_mask());
-    res->set_next_index(_buckets[ind]);
-    _buckets[ind] = new_ind;
-    _occupied_entries++;
-  }
-  return res;
-}
-
-int RSHashTable::alloc_entry() {
-  int res;
-  if (_free_list != NullEntry) {
-    res = _free_list;
-    _free_list = entry(res)->next_index();
-    return res;
-  } else if ((size_t)_free_region < _num_entries) {
-    res = _free_region;
-    _free_region++;
-    return res;
-  } else {
-    return NullEntry;
-  }
-}
-
-void RSHashTable::free_entry(int fi) {
-  entry(fi)->set_next_index(_free_list);
-  _free_list = fi;
-}
-
-void RSHashTable::add_entry(SparsePRTEntry* e) {
-  assert(e->num_valid_cards() > 0, "Precondition.");
-  SparsePRTEntry* e2 = entry_for_region_ind_create(e->r_ind());
-  e->copy_cards(e2);
-  assert(e2->num_valid_cards() > 0, "Postcondition.");
-}
-
-bool RSHashTableBucketIter::has_next(SparsePRTEntry*& entry) {
-  while (_bl_ind == RSHashTable::NullEntry)  {
-    if (_tbl_ind + 1 >= _rsht->capacity()) {
-      return false;
-    }
-    _tbl_ind++;
-    _bl_ind = _rsht->_buckets[_tbl_ind];
-  }
-  entry = _rsht->entry(_bl_ind);
-  _bl_ind = entry->next_index();
-  return true;
-}
-
-bool RSHashTable::contains_card(RegionIdx_t region_index, CardIdx_t card_index) const {
-  SparsePRTEntry* e = get_entry(region_index);
-  return (e != NULL && e->contains_card(card_index));
-}
-
-size_t RSHashTable::mem_size() const {
-  return sizeof(RSHashTable) +
-    _num_entries * (SparsePRTEntry::size() + sizeof(int));
-}
-
-// ----------------------------------------------------------------------
-
-SparsePRT::SparsePRT() :
-  _table(&RSHashTable::empty_table) {
-}
-
-
-SparsePRT::~SparsePRT() {
-  if (_table != &RSHashTable::empty_table) {
-    delete _table;
-  }
-}
-
-
-size_t SparsePRT::mem_size() const {
-  // We ignore "_cur" here, because it either = _next, or else it is
-  // on the deleted list.
-  return sizeof(SparsePRT) + _table->mem_size();
-}
-
-SparsePRT::AddCardResult SparsePRT::add_card(RegionIdx_t region_id, CardIdx_t card_index) {
-  if (_table->should_expand()) {
-    expand();
-  }
-  return _table->add_card(region_id, card_index);
-}
-
-SparsePRTEntry* SparsePRT::get_entry(RegionIdx_t region_id) {
-  return _table->get_entry(region_id);
-}
-
-bool SparsePRT::delete_entry(RegionIdx_t region_id) {
-  return _table->delete_entry(region_id);
-}
-
-void SparsePRT::clear() {
-  // If the entry table not at initial capacity, just reset to the empty table.
-  if (_table->capacity() == InitialCapacity) {
-    _table->clear();
-  } else if (_table != &RSHashTable::empty_table) {
-    delete _table;
-    _table = &RSHashTable::empty_table;
-  }
-}
-
-void SparsePRT::expand() {
-  RSHashTable* last = _table;
-  if (last != &RSHashTable::empty_table) {
-    _table = new RSHashTable(last->capacity() * 2);
-    for (size_t i = 0; i < last->num_entries(); i++) {
-      SparsePRTEntry* e = last->entry((int)i);
-      if (e->valid_entry()) {
-        _table->add_entry(e);
-      }
-    }
-    delete last;
-  } else {
-    _table = new RSHashTable(InitialCapacity);
-  }
-}

src/hotspot/share/gc/g1/sparsePRT.hpp

@@ -1,246 +0,0 @@
-/*
- * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_GC_G1_SPARSEPRT_HPP
-#define SHARE_GC_G1_SPARSEPRT_HPP
-
-#include "gc/g1/g1CollectedHeap.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/shared/cardTableBarrierSet.hpp"
-#include "memory/allocation.hpp"
-#include "runtime/mutex.hpp"
-#include "utilities/align.hpp"
-#include "utilities/globalDefinitions.hpp"
-
-class RSHashTable;
-class SparsePRTEntry;
-
-// Sparse remembered set for a heap region (the "owning" region).  Maps
-// indices of other regions to short sequences of cards in the other region
-// that might contain pointers into the owner region.
-// Concurrent access to a SparsePRT must be serialized by some external mutex.
-class SparsePRT {
-  friend class SparsePRTBucketIter;
-
-  RSHashTable* _table;
-
-  static const size_t InitialCapacity = 8;
-
-  void expand();
-
-public:
-  SparsePRT();
-  ~SparsePRT();
-
-  size_t mem_size() const;
-
-  enum AddCardResult {
-    overflow, // The table is full, could not add the card to the table.
-    found,    // The card is already in the PRT.
-    added     // The card has been added.
-  };
-
-  // Attempts to ensure that the given card_index in the given region is in
-  // the sparse table.  If successful (because the card was already
-  // present, or because it was successfully added) returns "true".
-  // Otherwise, returns "false" to indicate that the addition would
-  // overflow the entry for the region.  The caller must transfer these
-  // entries to a larger-capacity representation.
-  AddCardResult add_card(RegionIdx_t region_id, CardIdx_t card_index);
-
-  // Return the pointer to the entry associated with the given region.
-  SparsePRTEntry* get_entry(RegionIdx_t region_ind);
-
-  // If there is an entry for "region_ind", removes it and return "true";
-  // otherwise returns "false."
-  bool delete_entry(RegionIdx_t region_ind);
-
-  // Clear the table, and reinitialize to initial capacity.
-  void clear();
-
-  bool contains_card(RegionIdx_t region_id, CardIdx_t card_index) const;
-};
-
-class SparsePRTEntry: public CHeapObj<mtGC> {
-public:
-  // The type of a card entry.
-  typedef uint16_t card_elem_t;
-
-private:
-  // We need to make sizeof(SparsePRTEntry) an even multiple of maximum member size,
-  // in order to force correct alignment that could otherwise cause SIGBUS errors
-  // when reading the member variables. This calculates the minimum number of card
-  // array elements required to get that alignment.
-  static const size_t card_array_alignment = sizeof(int) / sizeof(card_elem_t);
-
-  RegionIdx_t _region_ind;
-  int         _next_index;
-  int         _next_null;
-  // The actual cards stored in this array.
-  // WARNING: Don't put any data members beyond this line. Card array has, in fact, variable length.
-  // It should always be the last data member.
-  card_elem_t _cards[card_array_alignment];
-
-  // Copy the current entry's cards into "cards".
-  inline void copy_cards(card_elem_t* cards) const;
-public:
-  // Returns the size of the entry, used for entry allocation.
-  static size_t size() { return sizeof(SparsePRTEntry) + sizeof(card_elem_t) * (cards_num() - card_array_alignment); }
-  // Returns the size of the card array.
-  static int cards_num() {
-    return align_up((int)G1RSetSparseRegionEntries, (int)card_array_alignment);
-  }
-
-  // Set the region_ind to the given value, and delete all cards.
-  inline void init(RegionIdx_t region_ind);
-
-  RegionIdx_t r_ind() const { return _region_ind; }
-  bool valid_entry() const { return r_ind() >= 0; }
-
-  int next_index() const { return _next_index; }
-  int* next_index_addr() { return &_next_index; }
-  void set_next_index(int ni) { _next_index = ni; }
-
-  // Returns "true" iff the entry contains the given card index.
-  inline bool contains_card(CardIdx_t card_index) const;
-
-  // Returns the number of non-NULL card entries.
-  inline int num_valid_cards() const { return _next_null; }
-
-  inline SparsePRT::AddCardResult add_card(CardIdx_t card_index);
-
-  // Copy the current entry's cards into the "_card" array of "e."
-  inline void copy_cards(SparsePRTEntry* e) const;
-
-  card_elem_t* cards() { return _cards; }
-
-  inline CardIdx_t card(int i) const {
-    assert(i >= 0, "must be nonnegative");
-    assert(i < cards_num(), "range checking");
-    return (CardIdx_t)_cards[i];
-  }
-};
-
-class RSHashTable : public CHeapObj<mtGC> {
-
-  friend class RSHashTableBucketIter;
-
-  // Inverse maximum hash table occupancy used.
-  static float TableOccupancyFactor;
-
-  size_t _num_entries;
-
-  size_t _capacity;
-  size_t _capacity_mask;
-  size_t _occupied_entries;
-
-  SparsePRTEntry* _entries;
-  int* _buckets;
-  int  _free_region;
-  int  _free_list;
-
-  // Requires that the caller hold a lock preventing parallel modifying
-  // operations, and that the the table be less than completely full.  If
-  // an entry for "region_ind" is already in the table, finds it and
-  // returns its address; otherwise allocates, initializes, inserts and
-  // returns a new entry for "region_ind".
-  SparsePRTEntry* entry_for_region_ind_create(RegionIdx_t region_ind);
-
-  // Returns the index of the next free entry in "_entries".
-  int alloc_entry();
-  // Declares the entry "fi" to be free.  (It must have already been
-  // deleted from any bucket lists.
-  void free_entry(int fi);
-
-  // For the empty sentinel created at static initialization time
-  RSHashTable();
-
-public:
-  RSHashTable(size_t capacity);
-  ~RSHashTable();
-
-  static const int NullEntry = -1;
-  static RSHashTable empty_table;
-
-  bool should_expand() const { return _occupied_entries == _num_entries; }
-
-  // Attempts to ensure that the given card_index in the given region is in
-  // the sparse table.  If successful (because the card was already
-  // present, or because it was successfully added) returns "true".
-  // Otherwise, returns "false" to indicate that the addition would
-  // overflow the entry for the region.  The caller must transfer these
-  // entries to a larger-capacity representation.
-  SparsePRT::AddCardResult add_card(RegionIdx_t region_id, CardIdx_t card_index);
-
-  bool delete_entry(RegionIdx_t region_id);
-
-  bool contains_card(RegionIdx_t region_id, CardIdx_t card_index) const;
-
-  void add_entry(SparsePRTEntry* e);
-
-  SparsePRTEntry* get_entry(RegionIdx_t region_id) const;
-
-  void clear();
-
-  size_t capacity() const      { return _capacity; }
-  size_t capacity_mask() const { return _capacity_mask;  }
-  size_t mem_size() const;
-  // The number of SparsePRTEntry instances available.
-  size_t num_entries() const { return _num_entries; }
-
-  SparsePRTEntry* entry(int i) const {
-    assert(i >= 0 && (size_t)i < _num_entries, "precondition");
-    return (SparsePRTEntry*)((char*)_entries + SparsePRTEntry::size() * i);
-  }
-
-  void print();
-};
-
-// This is embedded in HRRS iterator.
-class RSHashTableBucketIter {
-  uint _tbl_ind;        // [0.._rsht->_capacity)
-  int  _bl_ind;         // [-1, 0.._rsht->_capacity)
-
-  RSHashTable* _rsht;
-
-public:
-  RSHashTableBucketIter(RSHashTable* rsht) :
-    _tbl_ind(0),
-    _bl_ind(rsht->_buckets[_tbl_ind]),
-    _rsht(rsht) { }
-
-  bool has_next(SparsePRTEntry*& entry);
-};
-
-class SparsePRTBucketIter: public RSHashTableBucketIter {
-public:
-  SparsePRTBucketIter(const SparsePRT* sprt) :
-    RSHashTableBucketIter(sprt->_table) {}
-
-  bool has_next(SparsePRTEntry*& entry) {
-    return RSHashTableBucketIter::has_next(entry);
-  }
-};
-
-#endif // SHARE_GC_G1_SPARSEPRT_HPP

src/hotspot/share/gc/shared/cardTable.hpp

@@ -235,8 +235,8 @@ public:
     card_size_in_words          = card_size / sizeof(HeapWord)
   };
 
-  static CardValue clean_card_val()          { return clean_card; }
-  static CardValue dirty_card_val()          { return dirty_card; }
+  static constexpr CardValue clean_card_val()          { return clean_card; }
+  static constexpr CardValue dirty_card_val()          { return dirty_card; }
   static intptr_t clean_card_row_val()   { return clean_card_row; }
 
   // Card marking array base (adjusted for heap low boundary)

src/hotspot/share/gc/shared/workerDataArray.hpp

@@ -34,7 +34,7 @@ template <class T>
 class WorkerDataArray  : public CHeapObj<mtGC> {
   friend class WDAPrinter;
 public:
-  static const uint MaxThreadWorkItems = 6;
+  static const uint MaxThreadWorkItems = 9;
 private:
   T*          _data;
   uint        _length;

src/hotspot/share/memory/allocation.hpp

@@ -124,6 +124,7 @@ class AllocatedObj {
   f(mtThreadStack,    "Thread Stack")                                                \
   f(mtCode,           "Code")        /* generated code                            */ \
   f(mtGC,             "GC")                                                          \
+  f(mtGCCardSet,      "GCCardSet")   /* G1 card set remembered set                */ \
   f(mtCompiler,       "Compiler")                                                    \
   f(mtJVMCI,          "JVMCI")                                                       \
   f(mtInternal,       "Internal")    /* memory used by VM, but does not belong to */ \

src/hotspot/share/prims/whitebox.cpp

@@ -99,7 +99,7 @@
 #include "gc/g1/g1ConcurrentMark.hpp"
 #include "gc/g1/g1ConcurrentMarkThread.inline.hpp"
 #include "gc/g1/heapRegionManager.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #endif // INCLUDE_G1GC
 #if INCLUDE_PARALLELGC
 #include "gc/parallel/parallelScavengeHeap.inline.hpp"

src/hotspot/share/runtime/arguments.cpp

@@ -533,6 +533,8 @@ static SpecialFlag const special_jvm_flags[] = {
 
   // -------------- Obsolete Flags - sorted by expired_in --------------
   { "CriticalJNINatives",           JDK_Version::jdk(16), JDK_Version::jdk(18), JDK_Version::jdk(19) },
+  { "G1RSetRegionEntries",          JDK_Version::undefined(), JDK_Version::jdk(18), JDK_Version::jdk(19) },
+  { "G1RSetSparseRegionEntries",    JDK_Version::undefined(), JDK_Version::jdk(18), JDK_Version::jdk(19) },
   { "AlwaysLockClassLoader",        JDK_Version::jdk(17), JDK_Version::jdk(18), JDK_Version::jdk(19) },
   { "UseBiasedLocking",             JDK_Version::jdk(15), JDK_Version::jdk(18), JDK_Version::jdk(19) },
   { "BiasedLockingStartupDelay",    JDK_Version::jdk(15), JDK_Version::jdk(18), JDK_Version::jdk(19) },

src/hotspot/share/utilities/concurrentHashTable.hpp

@@ -189,6 +189,10 @@ class ConcurrentHashTable : public CHeapObj<F> {
 
     Bucket* get_buckets() { return _buckets; }
     Bucket* get_bucket(size_t idx) { return &_buckets[idx]; }
+
+    size_t get_mem_size() {
+      return sizeof(*this) + _size * sizeof(Bucket);
+    }
   };
 
   // For materializing a supplied value.
@@ -384,8 +388,10 @@ class ConcurrentHashTable : public CHeapObj<F> {
 
   TableRateStatistics _stats_rate;
 
+  size_t get_mem_size(Thread* thread);
+
   size_t get_size_log2(Thread* thread);
-  size_t get_node_size() const { return sizeof(Node); }
+  static size_t get_node_size() { return sizeof(Node); }
   bool is_max_size_reached() { return _size_limit_reached; }
 
   // This means no paused bucket resize operation is going to resume

src/hotspot/share/utilities/concurrentHashTable.inline.hpp

@@ -37,8 +37,8 @@
 
 // 2^30 = 1G buckets
 #define SIZE_BIG_LOG2 30
-// 2^5  = 32 buckets
-#define SIZE_SMALL_LOG2 5
+// 2^2  = 4 buckets
+#define SIZE_SMALL_LOG2 2
 
 // Number from spinYield.hpp. In some loops SpinYield would be unfair.
 #define SPINPAUSES_PER_YIELD 8192
@@ -817,10 +817,7 @@ inline bool ConcurrentHashTable<CONFIG, F>::
   }
 
   _new_table = new InternalTable(_table->_log2_size + 1);
-
-  if (_new_table->_log2_size == _log2_size_limit) {
-    _size_limit_reached = true;
-  }
+  _size_limit_reached = _new_table->_log2_size == _log2_size_limit;
 
   return true;
 }
@@ -954,6 +951,7 @@ inline bool ConcurrentHashTable<CONFIG, F>::
 {
   Node* current_node = bucket->first();
   while (current_node != NULL) {
+    Prefetch::read(current_node->next(), 0);
     if (!visitor_f(current_node->value())) {
       return false;
     }
@@ -1032,6 +1030,14 @@ inline ConcurrentHashTable<CONFIG, F>::
   delete _table;
 }
 
+template <typename CONFIG, MEMFLAGS F>
+inline size_t ConcurrentHashTable<CONFIG, F>::
+  get_mem_size(Thread* thread)
+{
+  ScopedCS cs(thread, this);
+  return sizeof(*this) + _table->get_mem_size();
+}
+
 template <typename CONFIG, MEMFLAGS F>
 inline size_t ConcurrentHashTable<CONFIG, F>::
   get_size_log2(Thread* thread)
@@ -1135,8 +1141,6 @@ inline void ConcurrentHashTable<CONFIG, F>::
   // We only allow this method to be used during a safepoint.
   assert(SafepointSynchronize::is_at_safepoint(),
          "must only be called in a safepoint");
-  assert(Thread::current()->is_VM_thread(),
-         "should be in vm thread");
 
   // Here we skip protection,
   // thus no other thread may use this table at the same time.

test/hotspot/gtest/gc/g1/test_freeRegionList.cpp

@@ -60,11 +60,14 @@ TEST_VM(FreeRegionList, length) {
   MemRegion mr3(mr2.end(), HeapRegion::GrainWords);
   MemRegion mr4(mr3.end(), HeapRegion::GrainWords);
 
-  HeapRegion hr0(0, &bot, mr0);
-  HeapRegion hr1(1, &bot, mr1);
-  HeapRegion hr2(2, &bot, mr2);
-  HeapRegion hr3(3, &bot, mr3);
-  HeapRegion hr4(4, &bot, mr4);
+  G1CardSetConfiguration config;
+
+  HeapRegion hr0(0, &bot, mr0, &config);
+  HeapRegion hr1(1, &bot, mr1, &config);
+  HeapRegion hr2(2, &bot, mr2, &config);
+  HeapRegion hr3(3, &bot, mr3, &config);
+  HeapRegion hr4(4, &bot, mr4, &config);
+
   l.add_ordered(&hr1);
   l.add_ordered(&hr0);
   l.add_ordered(&hr3);

test/hotspot/gtest/gc/g1/test_g1CardSet.cpp

@@ -0,0 +1,476 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/g1/g1CardSet.inline.hpp"
+#include "gc/g1/g1CardSetContainers.hpp"
+#include "gc/g1/g1CardSetMemory.hpp"
+#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/shared/gcTraceTime.inline.hpp"
+#include "gc/shared/workgroup.hpp"
+#include "logging/log.hpp"
+#include "unittest.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+class G1CardSetTest : public ::testing::Test {
+
+  class G1CountCardsClosure : public G1CardSet::G1CardSetCardIterator {
+  public:
+    size_t _num_cards;
+
+    G1CountCardsClosure() : _num_cards(0) { }
+    void do_card(uint region_idx, uint card_idx) override {
+      _num_cards++;
+    }
+  };
+
+  static WorkGang* _workers;
+  static uint _max_workers;
+
+  static WorkGang* workers() {
+    if (_workers == NULL) {
+      _max_workers = os::processor_count();
+      _workers = new WorkGang("G1CardSetTest Work Gang", _max_workers, false, false);
+      _workers->initialize_workers();
+      _workers->update_active_workers(_max_workers);
+    }
+    return _workers;
+  }
+
+  // Check whether iteration agrees with the expected number of entries. If the
+  // add has been single-threaded, we can also check whether the occupied()
+  // (which is an estimate in that case) agrees.
+  static void check_iteration(G1CardSet* card_set,
+                              const size_t expected,
+                              const bool add_was_single_threaded = true);
+
+public:
+  G1CardSetTest() { }
+  ~G1CardSetTest() { }
+
+  static uint next_random(uint& seed, uint i) {
+    // Park–Miller random number generator
+    seed = (seed * 279470273u) % 0xfffffffb;
+    return (seed % i);
+  }
+
+  static void cardset_basic_test();
+  static void cardset_mt_test();
+
+  static void add_cards(G1CardSet* card_set, uint cards_per_region, uint* cards, uint num_cards, G1AddCardResult* results);
+  static void contains_cards(G1CardSet* card_set, uint cards_per_region, uint* cards, uint num_cards);
+
+  static void translate_cards(uint cards_per_region, uint region_idx, uint* cards, uint num_cards);
+
+  static void iterate_cards(G1CardSet* card_set, G1CardSet::G1CardSetCardIterator* cl);
+};
+
+WorkGang* G1CardSetTest::_workers = NULL;
+uint G1CardSetTest::_max_workers = 0;
+
+void G1CardSetTest::add_cards(G1CardSet* card_set, uint cards_per_region, uint* cards, uint num_cards, G1AddCardResult* results) {
+  for (uint i = 0; i < num_cards; i++) {
+
+    uint region_idx = cards[i] / cards_per_region;
+    uint card_idx = cards[i] % cards_per_region;
+
+    G1AddCardResult res = card_set->add_card(region_idx, card_idx);
+    if (results != NULL) {
+      ASSERT_TRUE(res == results[i]);
+    }
+  }
+}
+
+class G1CheckCardClosure : public G1CardSet::G1CardSetCardIterator {
+  G1CardSet* _card_set;
+
+  uint _cards_per_region;
+  uint* _cards_to_expect;
+  uint _num_cards;
+
+  bool _wrong_region_idx;
+
+public:
+  G1CheckCardClosure(G1CardSet* card_set, uint cards_per_region, uint* cards_to_expect, uint num_cards) :
+    _card_set(card_set),
+    _cards_per_region(cards_per_region),
+    _cards_to_expect(cards_to_expect),
+    _num_cards(num_cards),
+    _wrong_region_idx(false) {
+  }
+
+  void do_card(uint region_idx, uint card_idx) override {
+    uint card = _cards_per_region * region_idx + card_idx;
+    for (uint i = 0; i < _num_cards; i++) {
+      if (_cards_to_expect[i] == card) {
+        _cards_to_expect[i] = (uint)-1;
+      }
+    }
+  }
+
+  bool all_found() const {
+    bool all_good = true;
+    for (uint i = 0; i < _num_cards; i++) {
+      if (_cards_to_expect[i] != (uint)-1) {
+        log_error(gc)("Could not find card %u in region %u",
+                      _cards_to_expect[i] % _cards_per_region,
+                      _cards_to_expect[i] / _cards_per_region);
+        all_good = false;
+      }
+    }
+    return all_good;
+  }
+};
+
+void G1CardSetTest::contains_cards(G1CardSet* card_set, uint cards_per_region, uint* cards, uint num_cards) {
+  for (uint i = 0; i < num_cards; i++) {
+    uint region_idx = cards[i] / cards_per_region;
+    uint card_idx = cards[i] % cards_per_region;
+
+    ASSERT_TRUE(card_set->contains_card(region_idx, card_idx));
+  }
+
+  G1CheckCardClosure cl(card_set, cards_per_region, cards, num_cards);
+  card_set->iterate_cards(cl);
+
+  ASSERT_TRUE(cl.all_found());
+}
+
+// Offsets the card indexes in the cards array by the region_idx.
+void G1CardSetTest::translate_cards(uint cards_per_region, uint region_idx, uint* cards, uint num_cards) {
+  for (uint i = 0; i < num_cards; i++) {
+    cards[i] = cards_per_region * region_idx + cards[i];
+  }
+}
+
+class G1CountCardsOccupied : public G1CardSet::G1CardSetPtrIterator {
+  size_t _num_occupied;
+
+public:
+  G1CountCardsOccupied() : _num_occupied(0) { }
+
+  void do_cardsetptr(uint region_idx, size_t num_occupied, G1CardSet::CardSetPtr card_set) override {
+    _num_occupied += num_occupied;
+  }
+
+  size_t num_occupied() const { return _num_occupied; }
+};
+
+void G1CardSetTest::check_iteration(G1CardSet* card_set, const size_t expected, const bool single_threaded) {
+
+  class CheckIterator : public G1CardSet::G1CardSetCardIterator {
+  public:
+    G1CardSet* _card_set;
+    size_t _num_found;
+
+    CheckIterator(G1CardSet* card_set) : _card_set(card_set), _num_found(0) { }
+
+    void do_card(uint region_idx, uint card_idx) override {
+      ASSERT_TRUE(_card_set->contains_card(region_idx, card_idx));
+      _num_found++;
+    }
+  } cl(card_set);
+
+  card_set->iterate_cards(cl);
+
+  ASSERT_TRUE(expected == cl._num_found);
+  // We can assert this only if we are single-threaded.
+  if (single_threaded) {
+    ASSERT_EQ(card_set->occupied(), cl._num_found);
+  }
+}
+
+void G1CardSetTest::cardset_basic_test() {
+
+  const uint CardsPerRegion = 2048;
+  const double FullCardSetThreshold = 0.8;
+  const double BitmapCoarsenThreshold = 0.9;
+
+  G1CardSetConfiguration config(log2i_exact(CardsPerRegion), 28, BitmapCoarsenThreshold, 8, FullCardSetThreshold, CardsPerRegion);
+  G1CardSetFreePool free_pool(config.num_mem_object_types());
+  G1CardSetMemoryManager mm(&config, &free_pool);
+
+  {
+    G1CardSet card_set(&config, &mm);
+
+    uint cards1[] = { 1, 2, 3 };
+    G1AddCardResult results1[] = { Added, Added, Added };
+    translate_cards(CardsPerRegion, 99, cards1, ARRAY_SIZE(cards1));
+    add_cards(&card_set, CardsPerRegion, cards1, ARRAY_SIZE(cards1), results1);
+    contains_cards(&card_set, CardsPerRegion, cards1, ARRAY_SIZE(cards1));
+    ASSERT_TRUE(card_set.occupied() == ARRAY_SIZE(cards1));
+
+    G1CountCardsClosure count_cards;
+    card_set.iterate_cards(count_cards);
+    ASSERT_TRUE(count_cards._num_cards == ARRAY_SIZE(cards1));
+
+    check_iteration(&card_set, card_set.occupied());
+
+    card_set.clear();
+    ASSERT_TRUE(card_set.occupied() == 0);
+
+    check_iteration(&card_set, 0);
+  }
+
+  {
+    G1CardSet card_set(&config, &mm);
+
+    uint cards1[] = { 0, 2047, 17, 17 };
+    G1AddCardResult results1[] = { Added, Added, Added, Found };
+    translate_cards(CardsPerRegion, 100, cards1, ARRAY_SIZE(cards1));
+    add_cards(&card_set, CardsPerRegion, cards1, ARRAY_SIZE(cards1), results1);
+    // -1 because of the duplicate at the end.
+    contains_cards(&card_set, CardsPerRegion, cards1, ARRAY_SIZE(cards1) - 1);
+    ASSERT_TRUE(card_set.occupied() == ARRAY_SIZE(cards1) - 1);
+
+    G1CountCardsClosure count_cards;
+    card_set.iterate_cards(count_cards);
+    ASSERT_TRUE(count_cards._num_cards == ARRAY_SIZE(cards1) - 1);
+
+    check_iteration(&card_set, card_set.occupied());
+
+    card_set.clear();
+    ASSERT_TRUE(card_set.occupied() == 0);
+  }
+
+  {
+    G1CardSet card_set(&config, &mm);
+
+    uint cards1[] = { 0, 2047, 17, 18 /* for region 100 */,
+                      1,  128, 35, 17 /* for region 990 */
+                    };
+    translate_cards(CardsPerRegion, 100, &cards1[0], 4);
+    translate_cards(CardsPerRegion, 990, &cards1[4], 4);
+
+    add_cards(&card_set, CardsPerRegion, cards1, ARRAY_SIZE(cards1), NULL);
+    contains_cards(&card_set, CardsPerRegion, cards1, ARRAY_SIZE(cards1));
+    ASSERT_TRUE(card_set.occupied() == ARRAY_SIZE(cards1));
+
+    G1CountCardsClosure count_cards;
+    card_set.iterate_cards(count_cards);
+    ASSERT_TRUE(count_cards._num_cards == ARRAY_SIZE(cards1));
+
+    check_iteration(&card_set, card_set.occupied());
+
+    card_set.clear();
+    ASSERT_TRUE(card_set.occupied() == 0);
+  }
+
+  {
+    G1CardSet card_set(&config, &mm);
+
+    uint cards1[100];
+    for (uint i = 0; i < ARRAY_SIZE(cards1); i++) {
+      cards1[i] = i + 3;
+      translate_cards(CardsPerRegion, i, &cards1[i], 1);
+    }
+    add_cards(&card_set, CardsPerRegion, cards1, ARRAY_SIZE(cards1), NULL);
+    contains_cards(&card_set, CardsPerRegion, cards1, ARRAY_SIZE(cards1));
+
+    ASSERT_TRUE(card_set.num_containers() == ARRAY_SIZE(cards1));
+    ASSERT_TRUE(card_set.occupied() == ARRAY_SIZE(cards1));
+
+    G1CountCardsClosure count_cards;
+    card_set.iterate_cards(count_cards);
+    ASSERT_TRUE(count_cards._num_cards == ARRAY_SIZE(cards1));
+
+    check_iteration(&card_set, card_set.occupied());
+
+    card_set.clear();
+    ASSERT_TRUE(card_set.occupied() == 0);
+  }
+
+  {
+    G1CardSet card_set(&config, &mm);
+
+    // Generate non-prime numbers from 1 to 1000
+    uint count = 0;
+    for (uint i = 2; i < 33; i++) {
+      if (!card_set.contains_card(100, i)) {
+        for (uint j = i * i; j < 1000; j += i) {
+          G1AddCardResult res = card_set.add_card(100, j);
+          count += (res == Added);
+        }
+      }
+    }
+
+    G1CountCardsOccupied cl;
+    card_set.iterate_containers(&cl);
+
+    ASSERT_TRUE(count == card_set.occupied());
+    ASSERT_TRUE(card_set.occupied() == cl.num_occupied());
+
+    check_iteration(&card_set, card_set.occupied());
+
+    card_set.clear();
+    ASSERT_TRUE(card_set.occupied() == 0);
+  }
+  { // Test coarsening to full
+    G1CardSet card_set(&config, &mm);
+
+    uint count = 0;
+    uint i = 10;
+    uint bitmap_threshold = config.cards_in_howl_bitmap_threshold();
+    for (; i <  bitmap_threshold + 10; i++) {
+      G1AddCardResult res = card_set.add_card(99, i);
+      ASSERT_TRUE(res == Added);
+      count++;
+      ASSERT_TRUE(count == card_set.occupied());
+    }
+
+    G1AddCardResult res = card_set.add_card(99, config.num_cards_in_howl_bitmap() - 1);
+    // Adding above card should have coarsened Bitmap -> Full.
+    ASSERT_TRUE(res == Added);
+    ASSERT_TRUE(config.num_cards_in_howl_bitmap() == card_set.occupied());
+
+    res = card_set.add_card(99, config.num_cards_in_howl_bitmap() - 2);
+    ASSERT_TRUE(res == Found);
+
+    uint threshold = config.cards_in_howl_threshold();
+    uint adjusted_threshold = config.cards_in_howl_bitmap_threshold() * config.num_buckets_in_howl();
+    i = config.num_cards_in_howl_bitmap();
+    count = i;
+    for (; i <  threshold; i++) {
+      G1AddCardResult res = card_set.add_card(99, i);
+      ASSERT_TRUE(res == Added);
+      count++;
+      ASSERT_TRUE(count == card_set.occupied());
+    }
+
+    res = card_set.add_card(99, CardsPerRegion - 1);
+    // Adding above card should have coarsened Howl -> Full.
+    ASSERT_TRUE(res == Added);
+    ASSERT_TRUE(CardsPerRegion == card_set.occupied());
+
+    check_iteration(&card_set, card_set.occupied());
+
+    res = card_set.add_card(99, CardsPerRegion - 2);
+    ASSERT_TRUE(res == Found);
+
+    G1CountCardsClosure count_cards;
+    card_set.iterate_cards(count_cards);
+    ASSERT_TRUE(count_cards._num_cards == config.max_cards_in_region());
+
+    card_set.clear();
+    ASSERT_TRUE(card_set.occupied() == 0);
+  }
+}
+
+class G1CardSetMtTestTask : public AbstractGangTask {
+  G1CardSet* _card_set;
+
+  size_t _added;
+  size_t _found;
+
+public:
+  G1CardSetMtTestTask(G1CardSet* card_set) :
+    AbstractGangTask(""),
+    _card_set(card_set),
+    _added(0),
+    _found(0) { }
+
+  void work(uint worker_id) {
+    uint seed = worker_id;
+    size_t added = 0;
+    size_t found = 0;
+
+    for (uint i = 0; i < 100000; i++) {
+      uint region = G1CardSetTest::next_random(seed, 1000);
+      uint card = G1CardSetTest::next_random(seed, 10000);
+
+      G1AddCardResult res = _card_set->add_card(region, card);
+
+      ASSERT_TRUE(res == Added || res == Found);
+      if (res == Added) {
+        added++;
+      } else if (res == Found) {
+        found++;
+      }
+    }
+    Atomic::add(&_added, added);
+    Atomic::add(&_found, found);
+  }
+
+  size_t added() const { return _added; }
+  size_t found() const { return _found; }
+};
+
+void G1CardSetTest::cardset_mt_test() {
+  const uint CardsPerRegion = 16384;
+  const double FullCardSetThreshold = 1.0;
+  const uint BitmapCoarsenThreshold = 1.0;
+
+  G1CardSetConfiguration config(log2i_exact(CardsPerRegion), 120, BitmapCoarsenThreshold, 8, FullCardSetThreshold, CardsPerRegion);
+  G1CardSetFreePool free_pool(config.num_mem_object_types());
+  G1CardSetMemoryManager mm(&config, &free_pool);
+
+  G1CardSet card_set(&config, &mm);
+
+  const uint num_workers = workers()->active_workers();
+
+  G1CardSetMtTestTask cl(&card_set);
+
+  {
+    GCTraceTime(Error, gc) x("Cardset test");
+    _workers->run_task(&cl, num_workers);
+  }
+
+  size_t num_found = 0;
+  // Now check the contents of the card set.
+  for (uint i = 0; i < num_workers; i++) {
+    uint seed = i;
+
+    for (uint j = 0; j < 100000; j++) {
+      uint region = G1CardSetTest::next_random(seed, 1000);
+      uint card = G1CardSetTest::next_random(seed, 10000);
+
+      bool contains = card_set.contains_card(region, card);
+      ASSERT_TRUE(contains);
+
+      num_found += contains;
+    }
+  }
+
+  ASSERT_TRUE(num_found == cl.added() + cl.found());
+
+  G1CountCardsClosure count_cards;
+  card_set.iterate_cards(count_cards);
+
+  check_iteration(&card_set, count_cards._num_cards, false /* add_was_single_threaded */);
+
+  // During coarsening we try to unblock concurrent threads as soon as possible,
+  // so we do not add the cards from the smaller CardSetContainer to the larger
+  // one immediately, allowing addition by concurrent threads after allocating
+  // the space immediately. So the amount of "successfully added" results may be
+  // (and in case of many threads typically is) higher than the number of unique
+  // cards.
+  ASSERT_TRUE(count_cards._num_cards <= cl.added());
+}
+
+TEST_VM(G1CardSetTest, basic_cardset_test) {
+  G1CardSetTest::cardset_basic_test();
+}
+
+TEST_VM(G1CardSetTest, mt_cardset_test) {
+  G1CardSetTest::cardset_mt_test();
+}

test/hotspot/gtest/gc/g1/test_g1CardSetContainers.cpp

@@ -0,0 +1,263 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/g1/g1CardSetContainers.inline.hpp"
+#include "gc/g1/heapRegionBounds.inline.hpp"
+#include "gc/shared/cardTable.hpp"
+#include "memory/allocation.inline.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/powerOfTwo.hpp"
+#include "unittest.hpp"
+
+class G1CardSetContainersTest : public ::testing::Test {
+public:
+  G1CardSetContainersTest() { }
+  ~G1CardSetContainersTest() { }
+
+  static uint cards_per_inlineptr_set(uint bits_per_card) {
+    return G1CardSetInlinePtr::max_cards_in_inline_ptr(bits_per_card);
+  }
+
+  static void cardset_inlineptr_test(uint bits_per_card);
+  static void cardset_array_test(uint cards_per_array);
+  static void cardset_bitmap_test(uint threshold, uint size_in_bits);
+};
+
+class G1FindCardsInRange : public StackObj {
+  uint _num_cards;
+  uint _range_min;
+  bool* _cards_found;
+public:
+  G1FindCardsInRange(uint range_min, uint range_max) :
+    _num_cards(range_max - range_min + 1),
+    _range_min(range_min),
+    _cards_found(NEW_C_HEAP_ARRAY(bool, _num_cards, mtGC)) {
+    for (uint i = 0; i < _num_cards; i++) {
+      _cards_found[i] = false;
+    }
+  }
+
+  void verify_all_found() {
+    verify_part_found(_num_cards);
+  }
+
+  void verify_part_found(uint num) {
+    for (uint i = 0; i < num; i++) {
+      ASSERT_TRUE(_cards_found[i]);
+    }
+  }
+
+  ~G1FindCardsInRange() {
+    FREE_C_HEAP_ARRAY(mtGC, _cards_found);
+  }
+  void operator()(uint card) {
+    ASSERT_TRUE((card - _range_min) < _num_cards);
+    ASSERT_FALSE(_cards_found[card - _range_min]); // Must not have been found yet.
+    _cards_found[card - _range_min] = true;
+  }
+};
+
+void G1CardSetContainersTest::cardset_inlineptr_test(uint bits_per_card) {
+  const uint CardsPerSet = cards_per_inlineptr_set(bits_per_card);
+
+  G1AddCardResult res;
+
+  G1CardSet::CardSetPtr value = G1CardSetInlinePtr();
+
+  for (uint i = 0; i < CardsPerSet; i++) {
+    {
+      G1CardSetInlinePtr cards(&value, value);
+      res = cards.add(i + 1, bits_per_card, CardsPerSet);
+      ASSERT_TRUE(res == Added);
+    }
+    {
+      G1CardSetInlinePtr cards(&value, value);
+      ASSERT_TRUE(cards.contains(i + 1, bits_per_card));
+    }
+  }
+
+  for (uint i = 0; i < CardsPerSet; i++) {
+    G1CardSetInlinePtr cards(value);
+    ASSERT_TRUE(cards.contains(i + 1, bits_per_card));
+  }
+
+  // Try to add again, should all return that the card had been added.
+  for (uint i = 0; i < CardsPerSet; i++) {
+    G1CardSetInlinePtr cards(&value, value);
+    res = cards.add(i + 1, bits_per_card, CardsPerSet);
+    ASSERT_TRUE(res == Found);
+  }
+
+  // Should be no more space in set.
+  {
+    G1CardSetInlinePtr cards(&value, value);
+    res = cards.add(CardsPerSet + 1, bits_per_card, CardsPerSet);
+    ASSERT_TRUE(res == Overflow);
+  }
+
+  // Cards should still be in the set.
+  for (uint i = 0; i < CardsPerSet; i++) {
+    G1CardSetInlinePtr cards(value);
+    ASSERT_TRUE(cards.contains(i + 1, bits_per_card));
+  }
+
+  // Boundary cards should not be in the set.
+  {
+    G1CardSetInlinePtr cards(value);
+    ASSERT_TRUE(!cards.contains(0, bits_per_card));
+    ASSERT_TRUE(!cards.contains(CardsPerSet + 1, bits_per_card));
+  }
+
+  // Verify iteration finds all cards too and only those.
+  {
+    G1FindCardsInRange found(1, CardsPerSet);
+    G1CardSetInlinePtr cards(value);
+    cards.iterate(found, bits_per_card);
+    found.verify_all_found();
+  }
+}
+
+void G1CardSetContainersTest::cardset_array_test(uint cards_per_array) {
+  uint8_t* cardset_data = NEW_C_HEAP_ARRAY(uint8_t, G1CardSetArray::size_in_bytes(cards_per_array), mtGC);
+  G1CardSetArray* cards = new (cardset_data) G1CardSetArray(1, cards_per_array);
+
+  ASSERT_TRUE(cards->contains(1)); // Added during initialization
+  ASSERT_TRUE(cards->num_entries() == 1); // Check it's the only one.
+
+  G1AddCardResult res;
+
+  // Add some elements
+  for (uint i = 1; i < cards_per_array; i++) {
+    res = cards->add(i + 1);
+    ASSERT_TRUE(res == Added);
+  }
+
+  // Check they are in the container.
+  for (uint i = 0; i < cards_per_array; i++) {
+    ASSERT_TRUE(cards->contains(i + 1));
+  }
+
+  // Try to add again, should all return that the card had been added.
+  for (uint i = 0; i < cards_per_array; i++) {
+    res = cards->add(i + 1);
+    ASSERT_TRUE(res == Found);
+  }
+
+  // Should be no more space in set.
+  {
+    res = cards->add(cards_per_array + 1);
+    ASSERT_TRUE(res == Overflow);
+  }
+
+  // Cards should still be in the set.
+  for (uint i = 0; i < cards_per_array; i++) {
+    ASSERT_TRUE(cards->contains(i + 1));
+  }
+
+  ASSERT_TRUE(!cards->contains(0));
+  ASSERT_TRUE(!cards->contains(cards_per_array + 1));
+
+  // Verify iteration finds all cards too.
+  {
+    G1FindCardsInRange found(1, cards_per_array);
+    cards->iterate(found);
+    found.verify_all_found();
+  }
+
+  FREE_C_HEAP_ARRAY(mtGC, cardset_data);
+}
+
+void G1CardSetContainersTest::cardset_bitmap_test(uint threshold, uint size_in_bits) {
+  uint8_t* cardset_data = NEW_C_HEAP_ARRAY(uint8_t, G1CardSetBitMap::size_in_bytes(size_in_bits), mtGC);
+  G1CardSetBitMap* cards = new (cardset_data) G1CardSetBitMap(1, size_in_bits);
+
+  ASSERT_TRUE(cards->contains(1, size_in_bits)); // Added during initialization
+  ASSERT_TRUE(cards->num_bits_set() == 1); // Should be the only one.
+
+  G1AddCardResult res;
+
+  for (uint i = 1; i < threshold; i++) {
+    res = cards->add(i + 1, threshold, size_in_bits);
+    ASSERT_TRUE(res == Added);
+  }
+
+  for (uint i = 0; i < threshold; i++) {
+    ASSERT_TRUE(cards->contains(i + 1, size_in_bits));
+  }
+
+  // Try to add again, should all return that the card had been added.
+  for (uint i = 0; i < threshold; i++) {
+    res = cards->add(i + 1, threshold, size_in_bits);
+    ASSERT_TRUE(res == Found);
+  }
+
+  // Should be no more space in set.
+  {
+    res = cards->add(threshold + 1, threshold, size_in_bits);
+    ASSERT_TRUE(res == Overflow);
+  }
+
+  // Cards should still be in the set.
+  for (uint i = 0; i < threshold; i++) {
+    ASSERT_TRUE(cards->contains(i + 1, size_in_bits));
+  }
+
+  ASSERT_TRUE(!cards->contains(0, size_in_bits));
+
+  // Verify iteration finds all cards too.
+  {
+    G1FindCardsInRange found(1, threshold + 1);
+    cards->iterate(found, size_in_bits, 0);
+    found.verify_part_found(threshold);
+  }
+
+  FREE_C_HEAP_ARRAY(mtGC, cardset_data);
+}
+
+TEST_VM_F(G1CardSetContainersTest, basic_cardset_inptr_test) {
+  uint const min = (uint)log2i(HeapRegionBounds::min_size());
+  uint const max = (uint)log2i(HeapRegionBounds::max_size());
+
+  for (uint i = min; i <= max; i++) {
+    G1CardSetContainersTest::cardset_inlineptr_test(i - CardTable::card_shift);
+  }
+}
+
+TEST_VM_F(G1CardSetContainersTest, basic_cardset_array_test) {
+  uint array_sizes[] = { 5, 9, 63, 77, 127 };
+
+  for (uint i = 0; i < ARRAY_SIZE(array_sizes); i++) {
+    size_t const max_cards_in_set = ARRAY_SIZE(array_sizes);
+    G1CardSetContainersTest::cardset_array_test(max_cards_in_set);
+  }
+}
+
+TEST_VM_F(G1CardSetContainersTest, basic_cardset_bitmap_test) {
+  uint bit_sizes[] = { 64, 2048 };
+  uint threshold_sizes[] = { 17, 330 };
+
+  for (uint i = 0; i < ARRAY_SIZE(bit_sizes); i++) {
+    G1CardSetContainersTest::cardset_bitmap_test(threshold_sizes[i], bit_sizes[i]);
+  }
+}

test/hotspot/jtreg/gc/arguments/TestG1RemSetFlags.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.arguments;
+
+/*
+ * @test TestG1RemSetFlags
+ * @requires vm.gc.G1
+ * @summary Verify that the remembered set flags are updated as expected
+ * @modules java.base/jdk.internal.misc
+ * @modules java.management/sun.management
+ * @library /test/lib
+ * @library /
+ * @run driver gc.arguments.TestG1RemSetFlags
+ */
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.process.ProcessTools;
+
+public class TestG1RemSetFlags {
+
+  private static void checkG1RemSetFlags(String[] flags, int exitValue) throws Exception {
+    ArrayList<String> flagList = new ArrayList<String>();
+    flagList.addAll(Arrays.asList(flags));
+    flagList.add("-XX:+UseG1GC");
+    flagList.add("-XX:+PrintFlagsFinal");
+    flagList.add("-version");
+
+    ProcessBuilder pb = GCArguments.createJavaProcessBuilder(flagList);
+    OutputAnalyzer output = new OutputAnalyzer(pb.start());
+    output.shouldHaveExitValue(exitValue);
+  }
+
+  public static void main(String args[]) throws Exception {
+    checkG1RemSetFlags(new String[] { "-XX:+UnlockExperimentalVMOptions", "-XX:G1RemSetHowlNumBuckets=8", "-XX:G1RemSetHowlMaxNumBuckets=8"  },  0);
+    checkG1RemSetFlags(new String[] { "-XX:+UnlockExperimentalVMOptions", "-XX:G1RemSetHowlNumBuckets=8", "-XX:G1RemSetHowlMaxNumBuckets=16"  },  0);
+    checkG1RemSetFlags(new String[] { "-XX:+UnlockExperimentalVMOptions", "-XX:G1RemSetHowlNumBuckets=16", "-XX:G1RemSetHowlMaxNumBuckets=8"  },  1);
+    checkG1RemSetFlags(new String[] { "-XX:+UnlockExperimentalVMOptions", "-XX:G1RemSetHowlNumBuckets=7"  },  1);
+    checkG1RemSetFlags(new String[] { "-XX:+UnlockExperimentalVMOptions", "-XX:G1RemSetHowlMaxNumBuckets=7"  },  1);
+  }
+}

test/hotspot/jtreg/gc/g1/TestGCLogMessages.java

@@ -101,9 +101,14 @@ public class TestGCLogMessages {
         new LogMessageWithLevel("Prepare Merge Heap Roots", Level.DEBUG),
         new LogMessageWithLevel("Eager Reclaim", Level.DEBUG),
         new LogMessageWithLevel("Remembered Sets", Level.DEBUG),
-        new LogMessageWithLevel("Merged Sparse", Level.DEBUG),
-        new LogMessageWithLevel("Merged Fine", Level.DEBUG),
-        new LogMessageWithLevel("Merged Coarse", Level.DEBUG),
+        new LogMessageWithLevel("Merged Inline", Level.DEBUG),
+        new LogMessageWithLevel("Merged ArrayOfCards", Level.DEBUG),
+        new LogMessageWithLevel("Merged Howl", Level.DEBUG),
+        new LogMessageWithLevel("Merged Full", Level.DEBUG),
+        new LogMessageWithLevel("Merged Howl Inline", Level.DEBUG),
+        new LogMessageWithLevel("Merged Howl ArrayOfCards", Level.DEBUG),
+        new LogMessageWithLevel("Merged Howl BitMap", Level.DEBUG),
+        new LogMessageWithLevel("Merged Howl Full", Level.DEBUG),
         new LogMessageWithLevel("Hot Card Cache", Level.DEBUG),
         new LogMessageWithLevel("Log Buffers", Level.DEBUG),
         new LogMessageWithLevel("Dirty Cards", Level.DEBUG),
@@ -135,6 +140,7 @@ public class TestGCLogMessages {
         new LogMessageWithLevel("Region Register", Level.DEBUG),
         new LogMessageWithLevel("Prepare Heap Roots", Level.DEBUG),
         new LogMessageWithLevel("Concatenate Dirty Card Logs", Level.DEBUG),
+        new LogMessageWithLevel("Sample Collection Set Candidates", Level.DEBUG),
         // Free CSet
         new LogMessageWithLevel("Free Collection Set", Level.DEBUG),
         new LogMessageWithLevel("Serial Free Collection Set", Level.TRACE),

test/hotspot/jtreg/gc/g1/TestNoEagerReclaimOfHumongousRegions.java

@@ -34,7 +34,7 @@ package gc.g1;
  * @modules java.base/jdk.internal.misc
  * @build sun.hotspot.WhiteBox
  * @run driver jdk.test.lib.helpers.ClassFileInstaller sun.hotspot.WhiteBox
- * @run main/othervm -Xbootclasspath/a:. -Xlog:gc,gc+humongous=debug -XX:+UseG1GC -XX:MaxTenuringThreshold=0 -XX:G1RSetSparseRegionEntries=32 -XX:G1HeapRegionSize=1m -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI gc.g1.TestNoEagerReclaimOfHumongousRegions
+ * @run main/othervm -Xbootclasspath/a:. -Xlog:gc,gc+humongous=debug -XX:+UseG1GC -XX:MaxTenuringThreshold=0 -XX:+UnlockExperimentalVMOptions -XX:G1RemSetArrayOfCardsEntries=32 -XX:G1HeapRegionSize=1m -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI gc.g1.TestNoEagerReclaimOfHumongousRegions
  */
 
 import java.util.LinkedList;

test/hotspot/jtreg/gc/stress/TestStressRSetCoarsening.java

@@ -102,6 +102,8 @@ public class TestStressRSetCoarsening {
     private static final long KB = 1024;
     private static final long MB = 1024 * KB;
 
+    private static final int CARDSIZE = 512; // Card size in bytes.
+
     private static final WhiteBox WB = WhiteBox.getWhiteBox();
 
     public final ObjStorage storage;
@@ -229,11 +231,13 @@ public class TestStressRSetCoarsening {
     }
 
     public void go() throws InterruptedException {
-        // threshold for sparce -> fine
-        final int FINE = WB.getIntxVMFlag("G1RSetSparseRegionEntries").intValue();
+        // Threshold for Array of Cards -> Howl
+        final int ARRAY_TO_HOWL_THRESHOLD = WB.getUintVMFlag("G1RemSetArrayOfCardsEntries").intValue();
 
-        // threshold for fine -> coarse
-        final int COARSE = WB.getIntxVMFlag("G1RSetRegionEntries").intValue();
+        // Threshold for Howl -> Full
+        int coarsenHowlToFullPercent = WB.getUintVMFlag("G1RemSetCoarsenHowlToFullPercent").intValue();
+        int cardsPerRegion = WB.getSizeTVMFlag("G1HeapRegionSize").intValue() / CARDSIZE;
+        final int HOWL_TO_FULL_THRESHOLD = (cardsPerRegion * coarsenHowlToFullPercent) / 100;
 
         // regToRegRefCounts - array of reference counts from region to region
         // at the the end of iteration.
@@ -241,8 +245,13 @@ public class TestStressRSetCoarsening {
         // If c[i] > c[i-1] then during the iteration i more references will
         // be created.
         // If c[i] < c[i-1] then some referenes will be cleaned.
-        int[] regToRegRefCounts = {0, FINE / 2, 0, FINE, (FINE + COARSE) / 2, 0,
-            COARSE, COARSE + 10, FINE + 1, FINE / 2, 0};
+        int[] regToRegRefCounts = {
+            0, ARRAY_TO_HOWL_THRESHOLD / 2,
+            0, ARRAY_TO_HOWL_THRESHOLD,
+            (ARRAY_TO_HOWL_THRESHOLD + HOWL_TO_FULL_THRESHOLD) / 2, 0,
+            HOWL_TO_FULL_THRESHOLD, HOWL_TO_FULL_THRESHOLD + 10,
+            ARRAY_TO_HOWL_THRESHOLD + 1, ARRAY_TO_HOWL_THRESHOLD / 2,
+            0};
 
         // For progress tracking
         int[] progress = new int[regToRegRefCounts.length];

test/hotspot/jtreg/runtime/CommandLine/OptionsValidation/TestOptionsWithRanges.java

@@ -230,8 +230,6 @@ public class TestOptionsWithRanges {
          */
         excludeTestMaxRange("ConcGCThreads");
         excludeTestMaxRange("G1ConcRefinementThreads");
-        excludeTestMaxRange("G1RSetRegionEntries");
-        excludeTestMaxRange("G1RSetSparseRegionEntries");
         excludeTestMaxRange("G1UpdateBufferSize");
         excludeTestMaxRange("InitialHeapSize");
         excludeTestMaxRange("MaxHeapSize");

test/hotspot/jtreg/runtime/NMT/MallocSiteTypeChange.java

@@ -50,7 +50,7 @@ public class MallocSiteTypeChange {
         long addr = wb.NMTMallocWithPseudoStack(4 * 1024, pc);
 
         // Verify that current tracking level is "detail"
-        pb.command(new String[] { JDKToolFinder.getJDKTool("jcmd"), pid, "VM.native_memory", "summary"});
+        pb.command(new String[] { JDKToolFinder.getJDKTool("jcmd"), pid, "VM.native_memory", "detail"});
         output = new OutputAnalyzer(pb.start());
         output.shouldContain("Test (reserved=4KB, committed=4KB)");
 
@@ -59,7 +59,7 @@ public class MallocSiteTypeChange {
         output.shouldContain("Baseline succeeded");
 
         wb.NMTFree(addr);
-        addr = wb.NMTMallocWithPseudoStackAndType(2 * 1024, pc, 8 /* mtInternal */ );
+        addr = wb.NMTMallocWithPseudoStackAndType(2 * 1024, pc, 9 /* mtInternal */ );
         pb.command(new String[] { JDKToolFinder.getJDKTool("jcmd"), pid, "VM.native_memory", "detail.diff"});
         output = new OutputAnalyzer(pb.start());
         output.shouldContain("(malloc=0KB type=Test -4KB)");

test/jdk/jdk/jfr/event/gc/collection/TestG1ParallelPhases.java

@@ -115,7 +115,8 @@ public class TestG1ParallelPhases {
             "MergePSS",
             "NonYoungFreeCSet",
             "YoungFreeCSet",
-            "RebuildFreeList"
+            "RebuildFreeList",
+            "SampleCandidates"
         );
 
         // Some GC phases may or may not occur depending on environment. Filter them out