8290525: Move HeapRegion::_compaction_top to G1FullCollector

Reviewed-by: kbarrett, sangheki

src/hotspot/share/gc/g1/g1FullCollector.cpp

@@ -119,7 +119,7 @@ G1FullCollector::G1FullCollector(G1CollectedHeap* heap,
     _oop_queue_set(_num_workers),
     _array_queue_set(_num_workers),
     _preserved_marks_set(true),
-    _serial_compaction_point(),
+    _serial_compaction_point(this),
     _is_alive(this, heap->concurrent_mark()->mark_bitmap()),
     _is_alive_mutator(heap->ref_processor_stw(), &_is_alive),
     _always_subject_to_discovery(),
@@ -132,13 +132,15 @@ G1FullCollector::G1FullCollector(G1CollectedHeap* heap,
   _compaction_points = NEW_C_HEAP_ARRAY(G1FullGCCompactionPoint*, _num_workers, mtGC);
 
   _live_stats = NEW_C_HEAP_ARRAY(G1RegionMarkStats, _heap->max_regions(), mtGC);
+  _compaction_tops = NEW_C_HEAP_ARRAY(HeapWord*, _heap->max_regions(), mtGC);
   for (uint j = 0; j < heap->max_regions(); j++) {
     _live_stats[j].clear();
+    _compaction_tops[j] = nullptr;
   }
 
   for (uint i = 0; i < _num_workers; i++) {
     _markers[i] = new G1FullGCMarker(this, i, _preserved_marks_set.get(i), _live_stats);
-    _compaction_points[i] = new G1FullGCCompactionPoint();
+    _compaction_points[i] = new G1FullGCCompactionPoint(this);
     _oop_queue_set.register_queue(i, marker(i)->oop_stack());
     _array_queue_set.register_queue(i, marker(i)->objarray_stack());
   }
@@ -152,6 +154,7 @@ G1FullCollector::~G1FullCollector() {
   }
   FREE_C_HEAP_ARRAY(G1FullGCMarker*, _markers);
   FREE_C_HEAP_ARRAY(G1FullGCCompactionPoint*, _compaction_points);
+  FREE_C_HEAP_ARRAY(HeapWord*, _compaction_tops);
   FREE_C_HEAP_ARRAY(G1RegionMarkStats, _live_stats);
 }
 
@@ -367,7 +370,7 @@ void G1FullCollector::phase2c_prepare_serial_compaction() {
     } else {
       assert(!current->is_humongous(), "Should be no humongous regions in compaction queue");
       G1SerialRePrepareClosure re_prepare(cp, current);
-      current->set_compaction_top(current->bottom());
+      set_compaction_top(current, current->bottom());
       current->apply_to_marked_objects(mark_bitmap(), &re_prepare);
     }
   }

src/hotspot/share/gc/g1/g1FullCollector.hpp

@@ -45,6 +45,7 @@ class G1FullGCMarker;
 class G1FullGCScope;
 class G1FullGCCompactionPoint;
 class GCMemoryManager;
+class HeapRegion;
 class ReferenceProcessor;
 
 // Subject-to-discovery closure for reference processing during Full GC. During
@@ -89,6 +90,8 @@ class G1FullCollector : StackObj {
 
   G1FullGCHeapRegionAttr _region_attr_table;
 
+  HeapWord* volatile* _compaction_tops;
+
 public:
   G1FullCollector(G1CollectedHeap* heap,
                   bool explicit_gc,
@@ -128,6 +131,9 @@ public:
   inline bool is_free(uint region_idx) const;
   inline void update_from_compacting_to_skip_compacting(uint region_idx);
 
+  inline void set_compaction_top(HeapRegion* r, HeapWord* value);
+  inline HeapWord* compaction_top(HeapRegion* r) const;
+
 private:
   void phase1_mark_live_objects();
   void phase2_prepare_compaction();

src/hotspot/share/gc/g1/g1FullCollector.inline.hpp

@@ -28,8 +28,9 @@
 #include "gc/g1/g1FullCollector.hpp"
 
 #include "gc/g1/g1FullGCHeapRegionAttr.hpp"
+#include "gc/g1/heapRegion.inline.hpp"
 #include "oops/oopsHierarchy.hpp"
-
+#include "runtime/atomic.hpp"
 
 bool G1FullCollector::is_compacting(oop obj) const {
   return _region_attr_table.is_compacting(cast_from_oop<HeapWord *>(obj));
@@ -60,5 +61,13 @@ void G1FullCollector::update_from_compacting_to_skip_compacting(uint region_idx)
   _region_attr_table.set_skip_compacting(region_idx);
 }
 
+void G1FullCollector::set_compaction_top(HeapRegion* r, HeapWord* value) {
+  Atomic::store(&_compaction_tops[r->hrm_index()], value);
+}
+
+HeapWord* G1FullCollector::compaction_top(HeapRegion* r) const {
+  return Atomic::load(&_compaction_tops[r->hrm_index()]);
+}
+
 #endif // SHARE_GC_G1_G1FULLCOLLECTOR_INLINE_HPP
 

src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp

@@ -25,7 +25,6 @@
 #include "precompiled.hpp"
 #include "gc/g1/g1CollectedHeap.hpp"
 #include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
-#include "gc/g1/g1FullCollector.hpp"
 #include "gc/g1/g1FullCollector.inline.hpp"
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/g1FullGCCompactTask.hpp"
@@ -63,6 +62,10 @@ public:
              "should be quite full");
     }
 #endif
+    assert(_collector->compaction_top(r) == nullptr,
+           "region %u compaction_top " PTR_FORMAT " must not be different from bottom " PTR_FORMAT,
+           r->hrm_index(), p2i(_collector->compaction_top(r)), p2i(r->bottom()));
+
     r->reset_skip_compacting_after_full_gc();
     return false;
   }
@@ -110,7 +113,7 @@ void G1FullGCCompactTask::compact_region(HeapRegion* hr) {
     hr->apply_to_marked_objects(collector()->mark_bitmap(), &compact);
   }
 
-  hr->reset_compacted_after_full_gc();
+  hr->reset_compacted_after_full_gc(_collector->compaction_top(hr));
 }
 
 void G1FullGCCompactTask::work(uint worker_id) {

src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp

@@ -33,17 +33,18 @@
 
 class G1CollectedHeap;
 class G1CMBitMap;
+class G1FullCollector;
 
 class G1FullGCCompactTask : public G1FullGCTask {
-protected:
+  G1FullCollector* _collector;
   HeapRegionClaimer _claimer;
 
-private:
   void compact_region(HeapRegion* hr);
 
 public:
   G1FullGCCompactTask(G1FullCollector* collector) :
     G1FullGCTask("G1 Compact Task", collector),
+    _collector(collector),
     _claimer(collector->workers()) { }
   void work(uint worker_id);
   void serial_compaction();

src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,14 +23,16 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/g1/g1FullCollector.inline.hpp"
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/heapRegion.hpp"
 #include "oops/oop.inline.hpp"
 #include "utilities/debug.hpp"
 
-G1FullGCCompactionPoint::G1FullGCCompactionPoint() :
-    _current_region(NULL),
-    _compaction_top(NULL) {
+G1FullGCCompactionPoint::G1FullGCCompactionPoint(G1FullCollector* collector) :
+    _collector(collector),
+    _current_region(nullptr),
+    _compaction_top(nullptr) {
   _compaction_regions = new (ResourceObj::C_HEAP, mtGC) GrowableArray<HeapRegion*>(32, mtGC);
   _compaction_region_iterator = _compaction_regions->begin();
 }
@@ -41,12 +43,12 @@ G1FullGCCompactionPoint::~G1FullGCCompactionPoint() {
 
 void G1FullGCCompactionPoint::update() {
   if (is_initialized()) {
-    _current_region->set_compaction_top(_compaction_top);
+    _collector->set_compaction_top(_current_region, _compaction_top);
   }
 }
 
 void G1FullGCCompactionPoint::initialize_values() {
-  _compaction_top = _current_region->compaction_top();
+  _compaction_top = _collector->compaction_top(_current_region);
 }
 
 bool G1FullGCCompactionPoint::has_regions() {
@@ -83,7 +85,7 @@ bool G1FullGCCompactionPoint::object_will_fit(size_t size) {
 
 void G1FullGCCompactionPoint::switch_region() {
   // Save compaction top in the region.
-  _current_region->set_compaction_top(_compaction_top);
+  _collector->set_compaction_top(_current_region, _compaction_top);
   // Get the next region and re-initialize the values.
   _current_region = next_region();
   initialize_values();

src/hotspot/share/gc/g1/g1FullGCCompactionPoint.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,9 +29,11 @@
 #include "oops/oopsHierarchy.hpp"
 #include "utilities/growableArray.hpp"
 
+class G1FullCollector;
 class HeapRegion;
 
 class G1FullGCCompactionPoint : public CHeapObj<mtGC> {
+  G1FullCollector* _collector;
   HeapRegion* _current_region;
   HeapWord*   _compaction_top;
   GrowableArray<HeapRegion*>* _compaction_regions;
@@ -43,7 +45,7 @@ class G1FullGCCompactionPoint : public CHeapObj<mtGC> {
   HeapRegion* next_region();
 
 public:
-  G1FullGCCompactionPoint();
+  G1FullGCCompactionPoint(G1FullCollector* collector);
   ~G1FullGCCompactionPoint();
 
   bool has_regions();

src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp

@@ -67,7 +67,7 @@ inline G1FullGCCompactionPoint* G1DetermineCompactionQueueClosure::next_compacti
 }
 
 inline void G1DetermineCompactionQueueClosure::add_to_compaction_queue(HeapRegion* hr) {
-  hr->set_compaction_top(hr->bottom());
+  _collector->set_compaction_top(hr, hr->bottom());
   G1FullGCCompactionPoint* cp = next_compaction_point();
   if (!cp->is_initialized()) {
     cp->initialize(hr);

src/hotspot/share/gc/g1/heapRegion.cpp

@@ -223,7 +223,6 @@ HeapRegion::HeapRegion(uint hrm_index,
   _bottom(mr.start()),
   _end(mr.end()),
   _top(NULL),
-  _compaction_top(NULL),
   _bot_part(bot, this),
   _pre_dummy_top(NULL),
   _rem_set(NULL),
@@ -258,7 +257,6 @@ void HeapRegion::initialize(bool clear_space, bool mangle_space) {
   }
 
   set_top(bottom());
-  set_compaction_top(bottom());
 
   hr_clear(false /*clear_space*/);
 }
@@ -768,7 +766,6 @@ void HeapRegion::verify_rem_set() const {
 
 void HeapRegion::clear(bool mangle_space) {
   set_top(bottom());
-  set_compaction_top(bottom());
 
   if (ZapUnusedHeapArea && mangle_space) {
     mangle_unused_area();

src/hotspot/share/gc/g1/heapRegion.hpp

@@ -73,7 +73,6 @@ class HeapRegion : public CHeapObj<mtGC> {
   HeapWord* const _end;
 
   HeapWord* volatile _top;
-  HeapWord* _compaction_top;
 
   G1BlockOffsetTablePart _bot_part;
 
@@ -89,9 +88,6 @@ public:
   HeapWord* bottom() const         { return _bottom; }
   HeapWord* end() const            { return _end;    }
 
-  void set_compaction_top(HeapWord* compaction_top) { _compaction_top = compaction_top; }
-  HeapWord* compaction_top() const { return _compaction_top; }
-
   void set_top(HeapWord* value) { _top = value; }
   HeapWord* top() const { return _top; }
 
@@ -124,7 +120,6 @@ public:
   bool is_empty() const { return used() == 0; }
 
 private:
-  void reset_compaction_top_after_compaction();
 
   void reset_after_full_gc_common();
 
@@ -184,7 +179,7 @@ public:
   void update_bot_for_block(HeapWord* start, HeapWord* end);
 
   // Update heap region that has been compacted to be consistent after Full GC.
-  void reset_compacted_after_full_gc();
+  void reset_compacted_after_full_gc(HeapWord* new_top);
   // Update skip-compacting heap region to be consistent after Full GC.
   void reset_skip_compacting_after_full_gc();
 

src/hotspot/share/gc/g1/heapRegion.inline.hpp

@@ -190,15 +190,10 @@ inline size_t HeapRegion::block_size(const HeapWord* p, HeapWord* const pb) cons
   return cast_to_oop(p)->size();
 }
 
-inline void HeapRegion::reset_compaction_top_after_compaction() {
-  set_top(compaction_top());
-  _compaction_top = bottom();
-}
-
-inline void HeapRegion::reset_compacted_after_full_gc() {
+inline void HeapRegion::reset_compacted_after_full_gc(HeapWord* new_top) {
   assert(!is_pinned(), "must be");
 
-  reset_compaction_top_after_compaction();
+  set_top(new_top);
   // After a compaction the mark bitmap in a non-pinned regions is invalid.
   // But all objects are live, we get this by setting TAMS to bottom.
   init_top_at_mark_start();
@@ -209,10 +204,6 @@ inline void HeapRegion::reset_compacted_after_full_gc() {
 inline void HeapRegion::reset_skip_compacting_after_full_gc() {
   assert(!is_free(), "must be");
 
-  assert(compaction_top() == bottom(),
-         "region %u compaction_top " PTR_FORMAT " must not be different from bottom " PTR_FORMAT,
-         hrm_index(), p2i(compaction_top()), p2i(bottom()));
-
   _marked_bytes = used();
   _garbage_bytes = 0;
 

src/hotspot/share/gc/g1/vmStructs_g1.hpp

@@ -41,7 +41,6 @@
   nonstatic_field(HeapRegion, _bottom,         HeapWord* const)               \
   nonstatic_field(HeapRegion, _top,            HeapWord* volatile)            \
   nonstatic_field(HeapRegion, _end,            HeapWord* const)               \
-  nonstatic_field(HeapRegion, _compaction_top, HeapWord*)                     \
                                                                               \
   nonstatic_field(HeapRegionType, _tag,       HeapRegionType::Tag volatile)   \
                                                                               \

src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/g1/HeapRegion.java

@@ -48,7 +48,6 @@ public class HeapRegion extends CompactibleSpace implements LiveRegionsProvider
     private static AddressField bottomField;
     private static AddressField topField;
     private static AddressField endField;
-    private static AddressField compactionTopField;
 
     private static CIntegerField grainBytesField;
     private static long typeFieldOffset;
@@ -70,7 +69,6 @@ public class HeapRegion extends CompactibleSpace implements LiveRegionsProvider
         bottomField = type.getAddressField("_bottom");
         topField = type.getAddressField("_top");
         endField = type.getAddressField("_end");
-        compactionTopField = type.getAddressField("_compaction_top");
 
         grainBytesField = type.getCIntegerField("GrainBytes");
         typeFieldOffset = type.getField("_type").getOffset();
@@ -93,8 +91,6 @@ public class HeapRegion extends CompactibleSpace implements LiveRegionsProvider
     public Address top()           { return topField.getValue(addr); }
     public Address end()           { return endField.getValue(addr); }
 
-    public Address compactionTop() { return compactionTopField.getValue(addr); }
-
     @Override
     public List<MemRegion> getLiveRegions() {
         List<MemRegion> res = new ArrayList<>();

src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shared/CompactibleSpace.java

@@ -39,7 +39,6 @@ import sun.jvm.hotspot.utilities.Observer;
     full GC's. */
 
 public abstract class CompactibleSpace extends Space {
-  private static AddressField compactionTopField;
 
   static {
     VM.registerVMInitializedObserver(new Observer() {
@@ -51,16 +50,9 @@ public abstract class CompactibleSpace extends Space {
 
   private static synchronized void initialize(TypeDataBase db) {
     Type type = db.lookupType("CompactibleSpace");
-
-    compactionTopField = type.getAddressField("_compaction_top");
   }
 
   public CompactibleSpace(Address addr) {
     super(addr);
   }
-
-  /** May be used temporarily during a compaction phase. */
-  public Address compactionTop() {
-    return compactionTopField.getValue(addr);
-  }
 }