8030646: track collection set membership in one place

Reviewed-by: tschatzl, jwilhelm

hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp

@@ -3538,7 +3538,7 @@ class RegisterHumongousWithInCSetFastTestClosure : public HeapRegionClosure {
         r->rem_set()->clear_locked();
       }
       assert(r->rem_set()->is_empty(), "At this point any humongous candidate remembered set must be empty.");
-      g1h->register_humongous_region_with_in_cset_fast_test(region_idx);
+      g1h->register_humongous_region_with_cset(region_idx);
       _candidate_humongous++;
     }
     _total_humongous++;
@@ -3552,7 +3552,7 @@ class RegisterHumongousWithInCSetFastTestClosure : public HeapRegionClosure {
   void flush_rem_set_entries() { _dcq.flush(); }
 };
 
-void G1CollectedHeap::register_humongous_regions_with_in_cset_fast_test() {
+void G1CollectedHeap::register_humongous_regions_with_cset() {
   if (!G1EagerReclaimHumongousObjects) {
     g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(0.0, 0, 0);
     return;
@@ -3859,7 +3859,7 @@ G1CollectedHeap::do_collection_pause_at_safepoint(double target_pause_time_ms) {
 
         g1_policy()->finalize_cset(target_pause_time_ms, evacuation_info);
 
-        register_humongous_regions_with_in_cset_fast_test();
+        register_humongous_regions_with_cset();
 
         assert(check_cset_fast_test(), "Inconsistency in the InCSetState table.");
 
@@ -6077,7 +6077,7 @@ void G1CollectedHeap::free_collection_set(HeapRegion* cs_head, EvacuationInfo& e
     HeapRegion* next = cur->next_in_collection_set();
     assert(cur->in_collection_set(), "bad CS");
     cur->set_next_in_collection_set(NULL);
-    cur->set_in_collection_set(false);
+    clear_in_cset(cur);
 
     if (cur->is_young()) {
       int index = cur->young_index_in_cset();
@@ -6303,7 +6303,7 @@ void G1CollectedHeap::abandon_collection_set(HeapRegion* cs_head) {
     HeapRegion* next = cur->next_in_collection_set();
     assert(cur->in_collection_set(), "bad CS");
     cur->set_next_in_collection_set(NULL);
-    cur->set_in_collection_set(false);
+    clear_in_cset(cur);
     cur->set_young_index_in_cset(-1);
     cur = next;
   }

hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp

@@ -645,23 +645,21 @@ public:
   // is considered a candidate for eager reclamation.
   bool humongous_region_is_candidate(uint index);
   // Register the given region to be part of the collection set.
-  inline void register_humongous_region_with_in_cset_fast_test(uint index);
+  inline void register_humongous_region_with_cset(uint index);
   // Register regions with humongous objects (actually on the start region) in
   // the in_cset_fast_test table.
-  void register_humongous_regions_with_in_cset_fast_test();
+  void register_humongous_regions_with_cset();
   // We register a region with the fast "in collection set" test. We
   // simply set to true the array slot corresponding to this region.
-  void register_young_region_with_in_cset_fast_test(HeapRegion* r) {
+  void register_young_region_with_cset(HeapRegion* r) {
     _in_cset_fast_test.set_in_young(r->hrm_index());
   }
-  void register_old_region_with_in_cset_fast_test(HeapRegion* r) {
+  void register_old_region_with_cset(HeapRegion* r) {
     _in_cset_fast_test.set_in_old(r->hrm_index());
   }
-
-  // This is a fast test on whether a reference points into the
-  // collection set or not. Assume that the reference
-  // points into the heap.
-  inline bool in_cset_fast_test(oop obj);
+  void clear_in_cset(const HeapRegion* hr) {
+    _in_cset_fast_test.clear(hr);
+  }
 
   void clear_cset_fast_test() {
     _in_cset_fast_test.clear();
@@ -1246,6 +1244,7 @@ public:
   // set. Slow implementation.
   inline bool obj_in_cs(oop obj);
 
+  inline bool is_in_cset(const HeapRegion *hr);
   inline bool is_in_cset(oop obj);
 
   inline bool is_in_cset_or_humongous(const oop obj);

hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp

@@ -234,6 +234,10 @@ inline bool G1CollectedHeap::is_in_cset(oop obj) {
   return ret;
 }
 
+bool G1CollectedHeap::is_in_cset(const HeapRegion* hr) {
+  return _in_cset_fast_test.is_in_cset(hr);
+}
+
 bool G1CollectedHeap::is_in_cset_or_humongous(const oop obj) {
   return _in_cset_fast_test.is_in_cset_or_humongous((HeapWord*)obj);
 }
@@ -242,7 +246,7 @@ InCSetState G1CollectedHeap::in_cset_state(const oop obj) {
   return _in_cset_fast_test.at((HeapWord*)obj);
 }
 
-void G1CollectedHeap::register_humongous_region_with_in_cset_fast_test(uint index) {
+void G1CollectedHeap::register_humongous_region_with_cset(uint index) {
   _in_cset_fast_test.set_humongous(index);
 }
 

hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp

@@ -1607,11 +1607,10 @@ void G1CollectorPolicy::add_old_region_to_cset(HeapRegion* hr) {
   assert(hr->is_old(), "the region should be old");
 
   assert(!hr->in_collection_set(), "should not already be in the CSet");
-  hr->set_in_collection_set(true);
+  _g1->register_old_region_with_cset(hr);
   hr->set_next_in_collection_set(_collection_set);
   _collection_set = hr;
   _collection_set_bytes_used_before += hr->used();
-  _g1->register_old_region_with_in_cset_fast_test(hr);
   size_t rs_length = hr->rem_set()->occupied();
   _recorded_rs_lengths += rs_length;
   _old_cset_region_length += 1;
@@ -1741,10 +1740,8 @@ void G1CollectorPolicy::add_region_to_incremental_cset_common(HeapRegion* hr) {
   _inc_cset_max_finger = MAX2(_inc_cset_max_finger, hr_end);
 
   assert(!hr->in_collection_set(), "invariant");
-  hr->set_in_collection_set(true);
-  assert( hr->next_in_collection_set() == NULL, "invariant");
-
-  _g1->register_young_region_with_in_cset_fast_test(hr);
+  _g1->register_young_region_with_cset(hr);
+  assert(hr->next_in_collection_set() == NULL, "invariant");
 }
 
 // Add the region at the RHS of the incremental cset

hotspot/src/share/vm/gc_implementation/g1/g1InCSetState.hpp

@@ -25,6 +25,7 @@
 #ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1INCSETSTATE_HPP
 #define SHARE_VM_GC_IMPLEMENTATION_G1_G1INCSETSTATE_HPP
 
+#include "gc_implementation/g1/heapRegion.hpp"
 #include "gc_implementation/g1/g1BiasedArray.hpp"
 #include "memory/allocation.hpp"
 
@@ -125,8 +126,10 @@ class G1InCSetStateFastTestBiasedMappedArray : public G1BiasedMappedArray<InCSet
 
   bool is_in_cset_or_humongous(HeapWord* addr) const { return at(addr).is_in_cset_or_humongous(); }
   bool is_in_cset(HeapWord* addr) const { return at(addr).is_in_cset(); }
+  bool is_in_cset(const HeapRegion* hr) const { return get_by_index(hr->hrm_index()).is_in_cset(); }
   InCSetState at(HeapWord* addr) const { return get_by_address(addr); }
   void clear() { G1BiasedMappedArray<InCSetState>::clear(); }
+  void clear(const HeapRegion* hr) { return set_by_index(hr->hrm_index(), InCSetState::NotInCSet); }
 };
 
 #endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1INCSETSTATE_HPP

hotspot/src/share/vm/gc_implementation/g1/heapRegion.cpp

@@ -162,7 +162,7 @@ void HeapRegion::hr_clear(bool par, bool clear_space, bool locked) {
          "we should have already filtered out humongous regions");
   assert(_end == orig_end(),
          "we should have already filtered out humongous regions");
-  assert(!_in_collection_set,
+  assert(!in_collection_set(),
          err_msg("Should not clear heap region %u in the collection set", hrm_index()));
 
   set_allocation_context(AllocationContext::system());
@@ -262,7 +262,6 @@ HeapRegion::HeapRegion(uint hrm_index,
     _hrm_index(hrm_index),
     _allocation_context(AllocationContext::system()),
     _humongous_start_region(NULL),
-    _in_collection_set(false),
     _next_in_special_set(NULL),
     _evacuation_failed(false),
     _prev_marked_bytes(0), _next_marked_bytes(0), _gc_efficiency(0.0),

hotspot/src/share/vm/gc_implementation/g1/heapRegion.hpp

@@ -236,8 +236,6 @@ class HeapRegion: public G1OffsetTableContigSpace {
 
   // For a humongous region, region in which it starts.
   HeapRegion* _humongous_start_region;
-  // True iff the region is in current collection_set.
-  bool _in_collection_set;
 
   // True iff an attempt to evacuate an object in the region failed.
   bool _evacuation_failed;
@@ -487,13 +485,8 @@ class HeapRegion: public G1OffsetTableContigSpace {
     return _rem_set;
   }
 
-  // True iff the region is in current collection_set.
-  bool in_collection_set() const {
-    return _in_collection_set;
-  }
-  void set_in_collection_set(bool b) {
-    _in_collection_set = b;
-  }
+  bool in_collection_set() const;
+
   HeapRegion* next_in_collection_set() {
     assert(in_collection_set(), "should only invoke on member of CS.");
     assert(_next_in_special_set == NULL ||

hotspot/src/share/vm/gc_implementation/g1/heapRegion.inline.hpp

@@ -196,4 +196,8 @@ inline void HeapRegion::note_end_of_copying(bool during_initial_mark) {
   }
 }
 
+inline bool HeapRegion::in_collection_set() const {
+  return G1CollectedHeap::heap()->is_in_cset(this);
+}
+
 #endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGION_INLINE_HPP