8329603: G1: Merge G1BlockOffsetTablePart into G1BlockOffsetTable

Reviewed-by: ayang, iwalulya

src/hotspot/share/cds/filemap.cpp

@@ -2227,11 +2227,11 @@ void FileMapInfo::fixup_mapped_heap_region() {
   if (ArchiveHeapLoader::is_mapped()) {
     assert(!_mapped_heap_memregion.is_empty(), "sanity");
 
-    // Populate the archive regions' G1BlockOffsetTableParts. That ensures
+    // Populate the archive regions' G1BlockOffsetTables. That ensures
-    // fast G1BlockOffsetTablePart::block_start operations for any given address
+    // fast G1BlockOffsetTable::block_start operations for any given address
     // within the archive regions when trying to find start of an object
     // (e.g. during card table scanning).
-    G1CollectedHeap::heap()->populate_archive_regions_bot_part(_mapped_heap_memregion);
+    G1CollectedHeap::heap()->populate_archive_regions_bot(_mapped_heap_memregion);
   }
 }
 

src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp

@@ -31,10 +31,6 @@
 #include "oops/oop.inline.hpp"
 #include "runtime/java.hpp"
 
-//////////////////////////////////////////////////////////////////////
-// G1BlockOffsetTable
-//////////////////////////////////////////////////////////////////////
-
 G1BlockOffsetTable::G1BlockOffsetTable(MemRegion heap, G1RegionToSpaceMapper* storage) :
   _reserved(heap), _offset_base(nullptr) {
 
@@ -62,14 +58,6 @@ void G1BlockOffsetTable::check_address(uint8_t* addr, const char* msg) const {
 }
 #endif // ASSERT
 
-//////////////////////////////////////////////////////////////////////
-// G1BlockOffsetTablePart
-//////////////////////////////////////////////////////////////////////
-
-G1BlockOffsetTablePart::G1BlockOffsetTablePart(G1BlockOffsetTable* array, HeapRegion* hr) :
-  _bot(array),
-  _hr(hr) {}
-
 // Write the backskip value for each region.
 //
 //    offset
@@ -102,10 +90,9 @@ G1BlockOffsetTablePart::G1BlockOffsetTablePart(G1BlockOffsetTable* array, HeapRe
 //      Move back N (e.g., 8) entries and repeat with the
 //        value of the new entry
 //
-void G1BlockOffsetTablePart::set_remainder_to_point_to_start_incl(uint8_t* start_card, uint8_t* end_card) {
+void G1BlockOffsetTable::set_remainder_to_point_to_start_incl(uint8_t* start_card, uint8_t* end_card) {
   assert(start_card <= end_card, "precondition");
-  assert(start_card > _bot->entry_for_addr(_hr->bottom()), "Cannot be first card");
+  assert(offset_array(start_card-1) < CardTable::card_size_in_words(),
-  assert(_bot->offset_array(start_card-1) < CardTable::card_size_in_words(),
          "Offset card has an unexpected value");
   uint8_t* start_card_for_region = start_card;
   uint8_t offset = UINT8_MAX;
@@ -116,11 +103,11 @@ void G1BlockOffsetTablePart::set_remainder_to_point_to_start_incl(uint8_t* start
     uint8_t* reach = start_card - 1 + (BOTConstants::power_to_cards_back(i+1) - 1);
     offset = CardTable::card_size_in_words() + i;
     if (reach >= end_card) {
-      _bot->set_offset_array(start_card_for_region, end_card, offset);
+      set_offset_array(start_card_for_region, end_card, offset);
       start_card_for_region = reach + 1;
       break;
     }
-    _bot->set_offset_array(start_card_for_region, reach, offset);
+    set_offset_array(start_card_for_region, reach, offset);
     start_card_for_region = reach + 1;
   }
   assert(start_card_for_region > end_card, "Sanity check");
@@ -131,36 +118,36 @@ void G1BlockOffsetTablePart::set_remainder_to_point_to_start_incl(uint8_t* start
 // The card-interval [start_card, end_card] is a closed interval; this
 // is an expensive check -- use with care and only under protection of
 // suitable flag.
-void G1BlockOffsetTablePart::check_all_cards(uint8_t* start_card, uint8_t* end_card) const {
+void G1BlockOffsetTable::check_all_cards(uint8_t* start_card, uint8_t* end_card) const {
   if (end_card < start_card) {
     return;
   }
-  guarantee(_bot->offset_array(start_card) == CardTable::card_size_in_words(), "Wrong value in second card");
+  guarantee(offset_array(start_card) == CardTable::card_size_in_words(), "Wrong value in second card");
   for (uint8_t* c = start_card + 1; c <= end_card; c++ /* yeah! */) {
-    uint8_t entry = _bot->offset_array(c);
+    uint8_t entry = offset_array(c);
     if ((unsigned)(c - start_card) > BOTConstants::power_to_cards_back(1)) {
       guarantee(entry > CardTable::card_size_in_words(),
                 "Should be in logarithmic region - "
                 "entry: %u, "
                 "_array->offset_array(c): %u, "
                 "N_words: %u",
-                (uint)entry, (uint)_bot->offset_array(c), CardTable::card_size_in_words());
+                (uint)entry, (uint)offset_array(c), CardTable::card_size_in_words());
     }
     size_t backskip = BOTConstants::entry_to_cards_back(entry);
     uint8_t* landing_card = c - backskip;
     guarantee(landing_card >= (start_card - 1), "Inv");
     if (landing_card >= start_card) {
-      guarantee(_bot->offset_array(landing_card) <= entry,
+      guarantee(offset_array(landing_card) <= entry,
                 "Monotonicity - landing_card offset: %u, "
                 "entry: %u",
-                (uint)_bot->offset_array(landing_card), (uint)entry);
+                (uint)offset_array(landing_card), (uint)entry);
     } else {
       guarantee(landing_card == start_card - 1, "Tautology");
       // Note that N_words is the maximum offset value
-      guarantee(_bot->offset_array(landing_card) < CardTable::card_size_in_words(),
+      guarantee(offset_array(landing_card) < CardTable::card_size_in_words(),
                 "landing card offset: %u, "
                 "N_words: %u",
-                (uint)_bot->offset_array(landing_card), (uint)CardTable::card_size_in_words());
+                (uint)offset_array(landing_card), (uint)CardTable::card_size_in_words());
     }
   }
 }
@@ -176,10 +163,9 @@ void G1BlockOffsetTablePart::check_all_cards(uint8_t* start_card, uint8_t* end_c
 //       ( ^    ]
 //         blk_start
 //
-void G1BlockOffsetTablePart::update_for_block_work(HeapWord* blk_start,
+void G1BlockOffsetTable::update_for_block_work(HeapWord* blk_start, HeapWord* blk_end) {
-                                                   HeapWord* blk_end) {
   HeapWord* const cur_card_boundary = align_up_by_card_size(blk_start);
-  uint8_t* const offset_card =  _bot->entry_for_addr(cur_card_boundary);
+  uint8_t* const offset_card = entry_for_addr(cur_card_boundary);
 
   assert(blk_start != nullptr && blk_end > blk_start,
          "phantom block");
@@ -191,16 +177,16 @@ void G1BlockOffsetTablePart::update_for_block_work(HeapWord* blk_start,
          "reference must be into the heap");
   assert(G1CollectedHeap::heap()->is_in_reserved(blk_end - 1),
          "limit must be within the heap");
-  assert(cur_card_boundary == _bot->addr_for_entry(offset_card),
+  assert(cur_card_boundary == addr_for_entry(offset_card),
          "Block offset table entry must agree with cur_card_boundary");
 
   // Mark the card that holds the offset into the block.
-  _bot->set_offset_array(offset_card, cur_card_boundary, blk_start);
+  set_offset_array(offset_card, cur_card_boundary, blk_start);
 
   // We need to now mark the subsequent cards that this block spans.
 
   // Index of card on which the block ends.
-  uint8_t* end_card = _bot->entry_for_addr(blk_end - 1);
+  uint8_t* end_card = entry_for_addr(blk_end - 1);
 
   // Are there more cards left to be updated?
   if (offset_card + 1 <= end_card) {
@@ -210,54 +196,53 @@ void G1BlockOffsetTablePart::update_for_block_work(HeapWord* blk_start,
 #ifdef ASSERT
   // Calculate new_card_boundary this way because end_index
   // may be the last valid index in the covered region.
-  HeapWord* new_card_boundary = _bot->addr_for_entry(end_card) + CardTable::card_size_in_words();
+  HeapWord* new_card_boundary = addr_for_entry(end_card) + CardTable::card_size_in_words();
   assert(new_card_boundary >= blk_end, "postcondition");
 
   // The offset can be 0 if the block starts on a boundary.  That
   // is checked by an assertion above.
-  uint8_t* previous_card = _bot->entry_for_addr(blk_start);
+  uint8_t* previous_card = entry_for_addr(blk_start);
-  HeapWord* boundary = _bot->addr_for_entry(previous_card);
+  HeapWord* boundary = addr_for_entry(previous_card);
-  assert((_bot->offset_array(offset_card) == 0 && blk_start == boundary) ||
+  assert((offset_array(offset_card) == 0 && blk_start == boundary) ||
-         (_bot->offset_array(offset_card) > 0 && _bot->offset_array(offset_card) < CardTable::card_size_in_words()),
+         (offset_array(offset_card) > 0 && offset_array(offset_card) < CardTable::card_size_in_words()),
          "offset array should have been set - "
          "index offset: %u, "
          "blk_start: " PTR_FORMAT ", "
          "boundary: " PTR_FORMAT,
-         (uint)_bot->offset_array(offset_card),
+         (uint)offset_array(offset_card),
          p2i(blk_start), p2i(boundary));
   for (uint8_t* j = offset_card + 1; j <= end_card; j++) {
-    assert(_bot->offset_array(j) > 0 &&
+    assert(offset_array(j) > 0 &&
-           _bot->offset_array(j) <=
+           offset_array(j) <= (uint8_t) (CardTable::card_size_in_words() + BOTConstants::N_powers - 1),
-             (uint8_t) (CardTable::card_size_in_words() + BOTConstants::N_powers - 1),
            "offset array should have been set - "
            "%u not > 0 OR %u not <= %u",
-           (uint) _bot->offset_array(j),
+           (uint) offset_array(j),
-           (uint) _bot->offset_array(j),
+           (uint) offset_array(j),
            (uint) (CardTable::card_size_in_words() + BOTConstants::N_powers - 1));
   }
 #endif
 }
 
-void G1BlockOffsetTablePart::verify() const {
+void G1BlockOffsetTable::verify(const HeapRegion* hr) const {
-  assert(_hr->bottom() < _hr->top(), "Only non-empty regions should be verified.");
+  assert(hr->bottom() < hr->top(), "Only non-empty regions should be verified.");
-  uint8_t* start_card = _bot->entry_for_addr(_hr->bottom());
+  uint8_t* start_card = entry_for_addr(hr->bottom());
-  uint8_t* end_card = _bot->entry_for_addr(_hr->top() - 1);
+  uint8_t* end_card = entry_for_addr(hr->top() - 1);
 
   for (uint8_t* current_card = start_card; current_card < end_card; current_card++) {
-    uint8_t entry = _bot->offset_array(current_card);
+    uint8_t entry = offset_array(current_card);
     if (entry < CardTable::card_size_in_words()) {
       // The entry should point to an object before the current card. Verify that
       // it is possible to walk from that object in to the current card by just
       // iterating over the objects following it.
-      HeapWord* card_address = _bot->addr_for_entry(current_card);
+      HeapWord* card_address = addr_for_entry(current_card);
       HeapWord* obj_end = card_address - entry;
       while (obj_end < card_address) {
         HeapWord* obj = obj_end;
-        size_t obj_size = _hr->block_size(obj);
+        size_t obj_size = hr->block_size(obj);
         obj_end = obj + obj_size;
-        guarantee(obj_end > obj && obj_end <= _hr->top(),
+        guarantee(obj_end > obj && obj_end <= hr->top(),
                   "Invalid object end. obj: " PTR_FORMAT " obj_size: " SIZE_FORMAT " obj_end: " PTR_FORMAT " top: " PTR_FORMAT,
-                  p2i(obj), obj_size, p2i(obj_end), p2i(_hr->top()));
+                  p2i(obj), obj_size, p2i(obj_end), p2i(hr->top()));
       }
     } else {
       // Because we refine the BOT based on which cards are dirty there is not much we can verify here.
@@ -271,31 +256,16 @@ void G1BlockOffsetTablePart::verify() const {
                 "Going backwards beyond the start_card. start_card: " SIZE_FORMAT " current_card: " SIZE_FORMAT " backskip: " SIZE_FORMAT,
                 p2i(start_card), p2i(current_card), backskip);
 
-      HeapWord* backskip_address = _bot->addr_for_entry(current_card - backskip);
+      HeapWord* backskip_address = addr_for_entry(current_card - backskip);
-      guarantee(backskip_address >= _hr->bottom(),
+      guarantee(backskip_address >= hr->bottom(),
                 "Going backwards beyond bottom of the region: bottom: " PTR_FORMAT ", backskip_address: " PTR_FORMAT,
-                p2i(_hr->bottom()), p2i(backskip_address));
+                p2i(hr->bottom()), p2i(backskip_address));
     }
   }
 }
 
-#ifndef PRODUCT
+void G1BlockOffsetTable::set_for_starts_humongous(HeapRegion* hr, HeapWord* obj_top, size_t fill_size) {
-void G1BlockOffsetTablePart::print_on(outputStream* out) {
+  update_for_block(hr->bottom(), obj_top);
-  uint8_t* from_card = _bot->entry_for_addr(_hr->bottom());
-  uint8_t* to_card = _bot->entry_for_addr(_hr->end());
-  out->print_cr(">> BOT for area [" PTR_FORMAT "," PTR_FORMAT ") "
-                "cards [" SIZE_FORMAT "," SIZE_FORMAT ")",
-                p2i(_hr->bottom()), p2i(_hr->end()), p2i(from_card), p2i(to_card));
-  for (uint8_t* i = from_card; i < to_card; ++i) {
-    out->print_cr("  entry " SIZE_FORMAT_W(8) " | " PTR_FORMAT " : %3u",
-                  p2i(i), p2i(_bot->addr_for_entry(i)),
-                  (uint) _bot->offset_array(i));
-  }
-}
-#endif // !PRODUCT
-
-void G1BlockOffsetTablePart::set_for_starts_humongous(HeapWord* obj_top, size_t fill_size) {
-  update_for_block(_hr->bottom(), obj_top);
   if (fill_size > 0) {
     update_for_block(obj_top, fill_size);
   }

src/hotspot/share/gc/g1/g1BlockOffsetTable.hpp

@@ -33,18 +33,13 @@
 #include "utilities/globalDefinitions.hpp"
 
 // Forward declarations
-class G1BlockOffsetTable;
 class HeapRegion;
 
 // This implementation of "G1BlockOffsetTable" divides the covered region
 // into "N"-word subregions (where "N" = 2^"LogN".  An array with an entry
 // for each such subregion indicates how far back one must go to find the
 // start of the chunk that includes the first word of the subregion.
-//
-// Each G1BlockOffsetTablePart is owned by a HeapRegion.
-
 class G1BlockOffsetTable: public CHeapObj<mtGC> {
-  friend class G1BlockOffsetTablePart;
   friend class VMStructs;
 
 private:
@@ -73,6 +68,23 @@ private:
 
   void check_address(uint8_t* addr, const char* msg) const NOT_DEBUG_RETURN;
 
+  // Sets the entries corresponding to the cards starting at "start" and ending
+  // at "end" to point back to the card before "start"; [start, end]
+  void set_remainder_to_point_to_start_incl(uint8_t* start, uint8_t* end);
+
+  // Update BOT entries corresponding to the mem range [blk_start, blk_end).
+  void update_for_block_work(HeapWord* blk_start, HeapWord* blk_end);
+
+  void check_all_cards(uint8_t* left_card, uint8_t* right_card) const NOT_DEBUG_RETURN;
+
+  static HeapWord* align_up_by_card_size(HeapWord* const addr) {
+    return align_up(addr, CardTable::card_size());
+  }
+
+  void update_for_block(HeapWord* blk_start, size_t size) {
+    update_for_block(blk_start, blk_start + size);
+  }
+
 public:
 
   // Return the number of slots needed for an offset array
@@ -96,35 +108,7 @@ public:
 
   // Mapping from object start array entry to address of first word
   HeapWord* addr_for_entry(const uint8_t* const p) const;
-};
 
-class G1BlockOffsetTablePart {
-  friend class G1BlockOffsetTable;
-  friend class VMStructs;
-private:
-  // This is the global BlockOffsetTable.
-  G1BlockOffsetTable* _bot;
-
-  // The region that owns this part of the BOT.
-  HeapRegion* _hr;
-
-  // Sets the entries corresponding to the cards starting at "start" and ending
-  // at "end" to point back to the card before "start"; [start, end]
-  void set_remainder_to_point_to_start_incl(uint8_t* start, uint8_t* end);
-
-  // Update BOT entries corresponding to the mem range [blk_start, blk_end).
-  void update_for_block_work(HeapWord* blk_start, HeapWord* blk_end);
-
-  void check_all_cards(uint8_t* left_card, uint8_t* right_card) const NOT_DEBUG_RETURN;
-
-  static HeapWord* align_up_by_card_size(HeapWord* const addr) {
-    return align_up(addr, CardTable::card_size());
-  }
-
-  void update_for_block(HeapWord* blk_start, size_t size) {
-    update_for_block(blk_start, blk_start + size);
-  }
-public:
   static bool is_crossing_card_boundary(HeapWord* const obj_start,
                                         HeapWord* const obj_end) {
     HeapWord* cur_card_boundary = align_up_by_card_size(obj_start);
@@ -132,10 +116,7 @@ public:
     return obj_end > cur_card_boundary;
   }
 
-  //  The elements of the array are initialized to zero.
+  void verify(const HeapRegion* hr) const;
-  G1BlockOffsetTablePart(G1BlockOffsetTable* array, HeapRegion* hr);
-
-  void verify() const;
 
   // Returns the address of the start of the block reaching into the card containing
   // "addr".
@@ -147,9 +128,7 @@ public:
     }
   }
 
-  void set_for_starts_humongous(HeapWord* obj_top, size_t fill_size);
+  void set_for_starts_humongous(HeapRegion* hr, HeapWord* obj_top, size_t fill_size);
-
-  void print_on(outputStream* out) PRODUCT_RETURN;
 };
 
 #endif // SHARE_GC_G1_G1BLOCKOFFSETTABLE_HPP

src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp

@@ -32,29 +32,20 @@
 #include "runtime/atomic.hpp"
 #include "oops/oop.inline.hpp"
 
-inline HeapWord* G1BlockOffsetTablePart::block_start_reaching_into_card(const void* addr) const {
+inline HeapWord* G1BlockOffsetTable::block_start_reaching_into_card(const void* addr) const {
-  assert(addr >= _hr->bottom() && addr < _hr->top(), "invalid address");
+  assert(_reserved.contains(addr), "invalid address");
 
-#ifdef ASSERT
+  uint8_t* entry = entry_for_addr(addr);
-  if (!_hr->is_continues_humongous()) {
+  uint8_t offset = offset_array(entry);
-    // For non-ContinuesHumongous regions, the first obj always starts from bottom.
-    uint8_t offset = _bot->offset_array(_bot->entry_for_addr(_hr->bottom()));
-    assert(offset == 0, "Found offset %u instead of 0 for region %u %s",
-           offset, _hr->hrm_index(), _hr->get_short_type_str());
-  }
-#endif
-
-  uint8_t* entry = _bot->entry_for_addr(addr);
-  uint8_t offset = _bot->offset_array(entry);
   while (offset >= CardTable::card_size_in_words()) {
     // The excess of the offset from N_words indicates a power of Base
     // to go back by.
     size_t n_cards_back = BOTConstants::entry_to_cards_back(offset);
     entry -= n_cards_back;
-    offset = _bot->offset_array(entry);
+    offset = offset_array(entry);
   }
   assert(offset < CardTable::card_size_in_words(), "offset too large");
-  HeapWord* q = _bot->addr_for_entry(entry);
+  HeapWord* q = addr_for_entry(entry);
   return q - offset;
 }
 

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -532,7 +532,7 @@ HeapWord* G1CollectedHeap::alloc_archive_region(size_t word_size, HeapWord* pref
   return start_addr;
 }
 
-void G1CollectedHeap::populate_archive_regions_bot_part(MemRegion range) {
+void G1CollectedHeap::populate_archive_regions_bot(MemRegion range) {
   assert(!is_init_completed(), "Expect to be called at JVM init time");
 
   iterate_regions_in_range(range,

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -717,9 +717,9 @@ public:
   // in the CDS archive.
   HeapWord* alloc_archive_region(size_t word_size, HeapWord* preferred_addr);
 
-  // Populate the G1BlockOffsetTablePart for archived regions with the given
+  // Populate the G1BlockOffsetTable for archived regions with the given
   // memory range.
-  void populate_archive_regions_bot_part(MemRegion range);
+  void populate_archive_regions_bot(MemRegion range);
 
   // For the specified range, uncommit the containing G1 regions
   // which had been allocated by alloc_archive_regions. This should be called

src/hotspot/share/gc/g1/g1HeapRegion.cpp

@@ -188,7 +188,7 @@ void HeapRegion::set_starts_humongous(HeapWord* obj_top, size_t fill_size) {
   _type.set_starts_humongous();
   _humongous_start_region = this;
 
-  _bot_part.set_for_starts_humongous(obj_top, fill_size);
+  _bot->set_for_starts_humongous(this, obj_top, fill_size);
 }
 
 void HeapRegion::set_continues_humongous(HeapRegion* first_hr) {
@@ -219,7 +219,7 @@ HeapRegion::HeapRegion(uint hrm_index,
   _bottom(mr.start()),
   _end(mr.end()),
   _top(nullptr),
-  _bot_part(bot, this),
+  _bot(bot),
   _pre_dummy_top(nullptr),
   _rem_set(nullptr),
   _hrm_index(hrm_index),
@@ -694,7 +694,7 @@ bool HeapRegion::verify(VerifyOption vo) const {
 
   // Only regions in old generation contain valid BOT.
   if (!is_empty() && !is_young()) {
-    _bot_part.verify();
+    _bot->verify(this);
   }
 
   if (is_humongous()) {
@@ -723,7 +723,7 @@ void HeapRegion::mangle_unused_area() {
 #endif
 
 void HeapRegion::update_bot_for_block(HeapWord* start, HeapWord* end) {
-  _bot_part.update_for_block(start, end);
+  _bot->update_for_block(start, end);
 }
 
 void HeapRegion::object_iterate(ObjectClosure* blk) {

src/hotspot/share/gc/g1/g1HeapRegion.hpp

@@ -74,7 +74,7 @@ class HeapRegion : public CHeapObj<mtGC> {
 
   HeapWord* volatile _top;
 
-  G1BlockOffsetTablePart _bot_part;
+  G1BlockOffsetTable* _bot;
 
   // When we need to retire an allocation region, while other threads
   // are also concurrently trying to allocate into it, we typically

src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp

@@ -100,12 +100,13 @@ inline HeapWord* HeapRegion::advance_to_block_containing_addr(const void* addr,
     cur_block = next_block;
     // Because the BOT is precise, we should never step into the next card
     // (i.e. crossing the card boundary).
-    assert(!G1BlockOffsetTablePart::is_crossing_card_boundary(cur_block, (HeapWord*)addr), "must be");
+    assert(!G1BlockOffsetTable::is_crossing_card_boundary(cur_block, (HeapWord*)addr), "must be");
   }
 }
 
 inline HeapWord* HeapRegion::block_start(const void* addr, HeapWord* const pb) const {
-  HeapWord* first_block = _bot_part.block_start_reaching_into_card(addr);
+  assert(addr >= bottom() && addr < top(), "invalid address");
+  HeapWord* first_block = _bot->block_start_reaching_into_card(addr);
   return advance_to_block_containing_addr(addr, pb, first_block);
 }
 
@@ -262,7 +263,7 @@ inline void HeapRegion::update_bot_for_obj(HeapWord* obj_start, size_t obj_size)
          HR_FORMAT_PARAMS(this),
          p2i(obj_start), p2i(obj_end));
 
-  _bot_part.update_for_block(obj_start, obj_end);
+  _bot->update_for_block(obj_start, obj_end);
 }
 
 inline HeapWord* HeapRegion::parsable_bottom() const {