8279008: G1: Calculate BOT threshold on-the-fly during Object Copy phase

Reviewed-by: tschatzl, sjohanss

src/hotspot/share/gc/g1/g1Allocator.cpp

@@ -301,10 +301,7 @@ G1PLABAllocator::G1PLABAllocator(G1Allocator* allocator) :
     _alloc_buffers[state] = NEW_C_HEAP_ARRAY(PLAB*, length, mtGC);
     size_t word_sz = _g1h->desired_plab_sz(state);
     for (uint node_index = 0; node_index < length; node_index++) {
-      // Specialized PLABs for old that handle BOT updates for object allocations.
+      _alloc_buffers[state][node_index] = new PLAB(word_sz);
-      _alloc_buffers[state][node_index] = (state == G1HeapRegionAttr::Old)
-                                        ? new G1BotUpdatingPLAB(word_sz)
-                                        : new PLAB(word_sz);
     }
   }
 }
@@ -364,7 +361,6 @@ HeapWord* G1PLABAllocator::allocate_direct_or_new_plab(G1HeapRegionAttr dest,
   // Try direct allocation.
   HeapWord* result = _allocator->par_allocate_during_gc(dest, word_sz, node_index);
   if (result != NULL) {
-    update_bot_for_direct_allocation(dest, result, word_sz);
     _direct_allocated[dest.type()] += word_sz;
   }
   return result;

src/hotspot/share/gc/g1/g1Allocator.hpp

@@ -145,22 +145,6 @@ public:
                                    uint node_index);
 };
 
-// Specialized PLAB for old generation promotions. For old regions the
-// BOT needs to be updated and the relevant data to do this efficiently
-// is stored in the PLAB.
-class G1BotUpdatingPLAB : public PLAB {
-  // An object spanning this threshold will cause a BOT update.
-  HeapWord* _next_bot_threshold;
-  // The region in which the PLAB resides.
-  HeapRegion* _region;
-public:
-  G1BotUpdatingPLAB(size_t word_sz) : PLAB(word_sz) { }
-  // Sets the new PLAB buffer as well as updates the threshold and region.
-  void set_buf(HeapWord* buf, size_t word_sz) override;
-  // Updates the BOT if the last allocation crossed the threshold.
-  inline void update_bot(size_t word_sz);
-};
-
 // Manages the PLABs used during garbage collection. Interface for allocation from PLABs.
 // Needs to handle multiple contexts, extra alignment in any "survivor" area and some
 // statistics.
@@ -181,18 +165,11 @@ private:
   inline PLAB* alloc_buffer(G1HeapRegionAttr dest, uint node_index) const;
   inline PLAB* alloc_buffer(region_type_t dest, uint node_index) const;
 
-  // Helpers to do explicit BOT updates for allocations in old generation regions.
-  void update_bot_for_direct_allocation(G1HeapRegionAttr attr, HeapWord* addr, size_t size);
-
   // Returns the number of allocation buffers for the given dest.
   // There is only 1 buffer for Old while Young may have multiple buffers depending on
   // active NUMA nodes.
   inline uint alloc_buffers_length(region_type_t dest) const;
 
-  // Returns if BOT updates are needed for the given destinaion. Currently we only have
-  // two destinations and BOT updates are only needed for the old generation.
-  inline bool needs_bot_update(G1HeapRegionAttr dest) const;
-
   bool may_throw_away_buffer(size_t const allocation_word_sz, size_t const buffer_size) const;
 public:
   G1PLABAllocator(G1Allocator* allocator);
@@ -221,9 +198,6 @@ public:
                             bool* refill_failed,
                             uint node_index);
 
-  // Update the BOT for the last PLAB allocation.
-  inline void update_bot_for_plab_allocation(G1HeapRegionAttr dest, size_t word_sz, uint node_index);
-
   void undo_allocation(G1HeapRegionAttr dest, HeapWord* obj, size_t word_sz, uint node_index);
 };
 

src/hotspot/share/gc/g1/g1Allocator.inline.hpp

@@ -133,49 +133,4 @@ inline HeapWord* G1PLABAllocator::allocate(G1HeapRegionAttr dest,
   return allocate_direct_or_new_plab(dest, word_sz, refill_failed, node_index);
 }
 
-inline bool G1PLABAllocator::needs_bot_update(G1HeapRegionAttr dest) const {
-  return dest.is_old();
-}
-
-inline void G1PLABAllocator::update_bot_for_direct_allocation(G1HeapRegionAttr attr, HeapWord* addr, size_t size) {
-  if (!needs_bot_update(attr)) {
-    return;
-  }
-
-  // Out of PLAB allocations in an old generation region. Update BOT.
-  HeapRegion* region = _g1h->heap_region_containing(addr);
-  region->update_bot_at(addr, size);
-}
-
-inline void G1PLABAllocator::update_bot_for_plab_allocation(G1HeapRegionAttr dest, size_t word_sz, uint node_index) {
-  assert(needs_bot_update(dest), "Wrong destination: %s", dest.get_type_str());
-  G1BotUpdatingPLAB* plab = static_cast<G1BotUpdatingPLAB*>(alloc_buffer(dest, node_index));
-  plab->update_bot(word_sz);
-}
-
-inline void G1BotUpdatingPLAB::set_buf(HeapWord* buf, size_t word_sz) {
-  PLAB::set_buf(buf, word_sz);
-  // Update the region and threshold to allow efficient BOT updates.
-  _region = G1CollectedHeap::heap()->heap_region_containing(buf);
-  _next_bot_threshold = _region->bot_threshold_for_addr(buf);
-}
-
-inline void G1BotUpdatingPLAB::update_bot(size_t word_sz) {
-  // The last object end is at _top, if it did not cross the
-  // threshold, there is nothing to do.
-  if (_top <= _next_bot_threshold) {
-    return;
-  }
-
-  HeapWord* obj_start = _top - word_sz;
-  assert(contains(obj_start),
-         "Object start outside PLAB. bottom: " PTR_FORMAT " object: " PTR_FORMAT,
-         p2i(_bottom), p2i(obj_start));
-  assert(obj_start <= _next_bot_threshold,
-         "Object start not below or at threshold. threshold: " PTR_FORMAT " object: " PTR_FORMAT,
-         p2i(_next_bot_threshold), p2i(obj_start));
-
-  _region->update_bot_crossing_threshold(&_next_bot_threshold, obj_start, _top);
-}
-
 #endif // SHARE_GC_G1_G1ALLOCATOR_INLINE_HPP

src/hotspot/share/gc/g1/g1BlockOffsetTable.hpp

@@ -160,8 +160,7 @@ public:
 
   void verify() const;
 
-  // Given an address calculate where the next threshold needing an update is.
+  inline HeapWord* align_up_by_card_size(HeapWord* const addr) const;
-  inline HeapWord* threshold_for_addr(const void* addr);
 
   // Returns the address of the start of the block containing "addr", or
   // else "null" if it is covered by no block.  (May have side effects,

src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp

@@ -31,18 +31,9 @@
 #include "gc/shared/memset_with_concurrent_readers.hpp"
 #include "runtime/atomic.hpp"
 
-inline HeapWord* G1BlockOffsetTablePart::threshold_for_addr(const void* addr) {
+inline HeapWord* G1BlockOffsetTablePart::align_up_by_card_size(HeapWord* const addr) const {
   assert(addr >= _hr->bottom() && addr < _hr->top(), "invalid address");
-  size_t index = _bot->index_for(addr);
+  return align_up(addr, BOTConstants::card_size());
-  HeapWord* card_boundary = _bot->address_for_index(index);
-  // Address at card boundary, use as threshold.
-  if (card_boundary == addr) {
-    return card_boundary;
-  }
-
-  // Calculate next threshold.
-  HeapWord* threshold = card_boundary + BOTConstants::card_size_in_words();
-  return threshold;
 }
 
 inline HeapWord* G1BlockOffsetTablePart::block_start(const void* addr) {

src/hotspot/share/gc/g1/g1ParScanThreadState.cpp

@@ -434,6 +434,12 @@ void G1ParScanThreadState::undo_allocation(G1HeapRegionAttr dest_attr,
   _plab_allocator->undo_allocation(dest_attr, obj_ptr, word_sz, node_index);
 }
 
+void G1ParScanThreadState::update_bot_after_copying(oop obj, size_t word_sz) {
+  HeapWord* obj_start = cast_from_oop<HeapWord*>(obj);
+  HeapRegion* region = _g1h->heap_region_containing(obj_start);
+  region->update_bot_if_crossing_boundary(obj_start, word_sz);
+}
+
 // Private inline function, for direct internal use and providing the
 // implementation of the public not-inline function.
 MAYBE_INLINE_EVACUATION
@@ -503,10 +509,7 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
       }
       _age_table.add(age, word_sz);
     } else {
-      // Currently we only have two destinations and we only need BOT updates for
+      update_bot_after_copying(obj, word_sz);
-      // old. If the current allocation was done outside the PLAB this call will
-      // have no effect since the _top of the PLAB has not changed.
-      _plab_allocator->update_bot_for_plab_allocation(dest_attr, word_sz, node_index);
     }
 
     // Most objects are not arrays, so do one array check rather than

src/hotspot/share/gc/g1/g1ParScanThreadState.hpp

@@ -169,6 +169,8 @@ private:
                        size_t word_sz,
                        uint node_index);
 
+  void update_bot_after_copying(oop obj, size_t word_sz);
+
   oop do_copy_to_survivor_space(G1HeapRegionAttr region_attr,
                                 oop obj,
                                 markWord old_mark);

src/hotspot/share/gc/g1/heapRegion.cpp

@@ -810,7 +810,7 @@ void HeapRegion::object_iterate(ObjectClosure* blk) {
 void HeapRegion::fill_with_dummy_object(HeapWord* address, size_t word_size, bool zap) {
   // Keep the BOT in sync for old generation regions.
   if (is_old()) {
-    update_bot_at(address, word_size);
+    update_bot_if_crossing_boundary(address, word_size);
   }
   // Fill in the object.
   CollectedHeap::fill_with_object(address, word_size, zap);

src/hotspot/share/gc/g1/heapRegion.hpp

@@ -162,13 +162,8 @@ public:
   inline HeapWord* allocate(size_t word_size);
   inline HeapWord* allocate(size_t min_word_size, size_t desired_word_size, size_t* actual_size);
 
-  // Update the BOT for the given address if it crosses the next
+  // Update BOT if this obj is the first entering a new card (i.e. crossing the card boundary).
-  // BOT threshold at or after obj_start.
+  inline void update_bot_if_crossing_boundary(HeapWord* obj_start, size_t obj_size);
-  inline void update_bot_at(HeapWord* obj_start, size_t obj_size);
-  // Update BOT at the given threshold for the given object. The
-  // given object must cross the threshold.
-  inline void update_bot_crossing_threshold(HeapWord** threshold, HeapWord* obj_start, HeapWord* obj_end);
-  inline HeapWord* bot_threshold_for_addr(const void* addr);
 
   // Full GC support methods.
 

src/hotspot/share/gc/g1/heapRegion.inline.hpp

@@ -231,32 +231,27 @@ inline HeapWord* HeapRegion::allocate(size_t min_word_size,
   return allocate_impl(min_word_size, desired_word_size, actual_word_size);
 }
 
-inline HeapWord* HeapRegion::bot_threshold_for_addr(const void* addr) {
+inline void HeapRegion::update_bot_if_crossing_boundary(HeapWord* obj_start, size_t obj_size) {
-  HeapWord* threshold = _bot_part.threshold_for_addr(addr);
-  assert(threshold >= addr,
-         "threshold must be at or after given address. " PTR_FORMAT " >= " PTR_FORMAT,
-         p2i(threshold), p2i(addr));
-  assert(is_old(),
-         "Should only calculate BOT threshold for old regions. addr: " PTR_FORMAT " region:" HR_FORMAT,
-         p2i(addr), HR_FORMAT_PARAMS(this));
-  return threshold;
-}
-
-inline void HeapRegion::update_bot_crossing_threshold(HeapWord** threshold, HeapWord* obj_start, HeapWord* obj_end) {
   assert(is_old(), "should only do BOT updates for old regions");
+
+  HeapWord* obj_end   = obj_start + obj_size;
+
   assert(is_in(obj_start), "obj_start must be in this region: " HR_FORMAT
-         " obj_start " PTR_FORMAT " obj_end " PTR_FORMAT " threshold " PTR_FORMAT,
+         " obj_start " PTR_FORMAT " obj_end " PTR_FORMAT,
          HR_FORMAT_PARAMS(this),
-         p2i(obj_start), p2i(obj_end), p2i(*threshold));
+         p2i(obj_start), p2i(obj_end));
-  _bot_part.alloc_block_work(threshold, obj_start, obj_end);
-}
 
-inline void HeapRegion::update_bot_at(HeapWord* obj_start, size_t obj_size) {
+  HeapWord* cur_card_boundary = _bot_part.align_up_by_card_size(obj_start);
-  HeapWord* threshold = bot_threshold_for_addr(obj_start);
-  HeapWord* obj_end = obj_start + obj_size;
 
-  if (obj_end > threshold) {
+  // strictly greater-than
-    update_bot_crossing_threshold(&threshold, obj_start, obj_end);
+  bool cross_card_boundary = (obj_end > cur_card_boundary);
+
+  if (cross_card_boundary) {
+    // Creating a dummy variable inside this `if` as the arg of `&`; this
+    // avoids unnecessary loads in the assembly code on the fast path (the
+    // bot-not-updating case).
+    HeapWord* dummy = cur_card_boundary;
+    _bot_part.alloc_block_work(&dummy, obj_start, obj_end);
   }
 }
 

src/hotspot/share/gc/shared/plab.hpp

@@ -116,7 +116,7 @@ public:
   }
 
   // Sets the space of the buffer to be [buf, space+word_sz()).
-  virtual void set_buf(HeapWord* buf, size_t new_word_sz) {
+  void set_buf(HeapWord* buf, size_t new_word_sz) {
     assert(new_word_sz > AlignmentReserve, "Too small");
     _word_sz = new_word_sz;