diff --git a/src/hotspot/share/gc/g1/g1AllocRegion.cpp b/src/hotspot/share/gc/g1/g1AllocRegion.cpp index a205cf71ee6..c4ab81cda8f 100644 --- a/src/hotspot/share/gc/g1/g1AllocRegion.cpp +++ b/src/hotspot/share/gc/g1/g1AllocRegion.cpp @@ -100,16 +100,13 @@ size_t G1AllocRegion::retire_internal(G1HeapRegion* alloc_region, bool fill_up) // it will never be empty. size_t waste = 0; assert_alloc_region(!alloc_region->is_empty(), - "the alloc region should never be empty"); + "the alloc region should never be empty"); if (fill_up) { waste = fill_up_remaining_space(alloc_region); } - assert_alloc_region(alloc_region->used() >= _used_bytes_before, "invariant"); - size_t allocated_bytes = alloc_region->used() - _used_bytes_before; - retire_region(alloc_region, allocated_bytes); - _used_bytes_before = 0; + retire_region(alloc_region); return waste; } @@ -132,15 +129,14 @@ size_t G1AllocRegion::retire(bool fill_up) { HeapWord* G1AllocRegion::new_alloc_region_and_allocate(size_t word_size) { assert_alloc_region(_alloc_region == _dummy_region, "pre-condition"); - assert_alloc_region(_used_bytes_before == 0, "pre-condition"); trace("attempting region allocation"); G1HeapRegion* new_alloc_region = allocate_new_region(word_size); if (new_alloc_region != nullptr) { new_alloc_region->reset_pre_dummy_top(); - // Need to do this before the allocation - _used_bytes_before = new_alloc_region->used(); - HeapWord* result = allocate(new_alloc_region, word_size); + + assert(new_alloc_region->is_empty(), "new regions should be empty"); + HeapWord* result = new_alloc_region->allocate(word_size); assert_alloc_region(result != nullptr, "the allocation should succeeded"); OrderAccess::storestore(); @@ -159,7 +155,7 @@ HeapWord* G1AllocRegion::new_alloc_region_and_allocate(size_t word_size) { void G1AllocRegion::init() { trace("initializing"); - assert_alloc_region(_alloc_region == nullptr && _used_bytes_before == 0, "pre-condition"); + assert_alloc_region(_alloc_region == nullptr, "pre-condition"); assert_alloc_region(_dummy_region != nullptr, "should have been set"); _alloc_region = _dummy_region; _count = 0; @@ -168,16 +164,9 @@ void G1AllocRegion::init() { void G1AllocRegion::set(G1HeapRegion* alloc_region) { trace("setting"); - // We explicitly check that the region is not empty to make sure we - // maintain the "the alloc region cannot be empty" invariant. - assert_alloc_region(alloc_region != nullptr && !alloc_region->is_empty(), "pre-condition"); - assert_alloc_region(_alloc_region == _dummy_region && - _used_bytes_before == 0 && _count == 0, - "pre-condition"); + assert_alloc_region(_alloc_region == _dummy_region && _count == 0, "pre-condition"); - _used_bytes_before = alloc_region->used(); - _alloc_region = alloc_region; - _count += 1; + update_alloc_region(alloc_region); trace("set"); } @@ -237,7 +226,7 @@ void G1AllocRegion::trace(const char* str, size_t min_word_size, size_t desired_ if (detailed_info) { if (result != nullptr) { out->print(" min " SIZE_FORMAT " desired " SIZE_FORMAT " actual " SIZE_FORMAT " " PTR_FORMAT, - min_word_size, desired_word_size, actual_word_size, p2i(result)); + min_word_size, desired_word_size, actual_word_size, p2i(result)); } else if (min_word_size != 0) { out->print(" min " SIZE_FORMAT " desired " SIZE_FORMAT, min_word_size, desired_word_size); } @@ -252,7 +241,6 @@ G1AllocRegion::G1AllocRegion(const char* name, uint node_index) : _alloc_region(nullptr), _count(0), - _used_bytes_before(0), _name(name), _node_index(node_index) { } @@ -261,9 +249,8 @@ G1HeapRegion* MutatorAllocRegion::allocate_new_region(size_t word_size) { return _g1h->new_mutator_alloc_region(word_size, _node_index); } -void MutatorAllocRegion::retire_region(G1HeapRegion* alloc_region, - size_t allocated_bytes) { - _g1h->retire_mutator_alloc_region(alloc_region, allocated_bytes); +void MutatorAllocRegion::retire_region(G1HeapRegion* alloc_region) { + _g1h->retire_mutator_alloc_region(alloc_region, alloc_region->used()); } void MutatorAllocRegion::init() { @@ -346,9 +333,11 @@ G1HeapRegion* G1GCAllocRegion::allocate_new_region(size_t word_size) { return _g1h->new_gc_alloc_region(word_size, _purpose, _node_index); } -void G1GCAllocRegion::retire_region(G1HeapRegion* alloc_region, - size_t allocated_bytes) { +void G1GCAllocRegion::retire_region(G1HeapRegion* alloc_region) { + assert(alloc_region->used() >= _used_bytes_before, "invariant"); + size_t allocated_bytes = alloc_region->used() - _used_bytes_before; _g1h->retire_gc_alloc_region(alloc_region, allocated_bytes, _purpose); + _used_bytes_before = 0; } size_t G1GCAllocRegion::retire(bool fill_up) { @@ -360,3 +349,8 @@ size_t G1GCAllocRegion::retire(bool fill_up) { } return end_waste; } + +void G1GCAllocRegion::reuse(G1HeapRegion* alloc_region) { + _used_bytes_before = alloc_region->used(); + set(alloc_region); +} diff --git a/src/hotspot/share/gc/g1/g1AllocRegion.hpp b/src/hotspot/share/gc/g1/g1AllocRegion.hpp index a8903fd54f1..391ded28388 100644 --- a/src/hotspot/share/gc/g1/g1AllocRegion.hpp +++ b/src/hotspot/share/gc/g1/g1AllocRegion.hpp @@ -63,11 +63,6 @@ private: // distinct regions this object can used during an active interval. uint _count; - // When we set up a new active region we save its used bytes in this - // field so that, when we retire it, we can calculate how much space - // we allocated in it. - size_t _used_bytes_before; - // Useful for debugging and tracing. const char* _name; @@ -94,24 +89,14 @@ protected: // The memory node index this allocation region belongs to. uint _node_index; + void set(G1HeapRegion* alloc_region); + // Reset the alloc region to point the dummy region. void reset_alloc_region(); - // Perform a non-MT-safe allocation out of the given region. - inline HeapWord* allocate(G1HeapRegion* alloc_region, - size_t word_size); - // Perform a MT-safe allocation out of the given region. inline HeapWord* par_allocate(G1HeapRegion* alloc_region, size_t word_size); - // Perform a MT-safe allocation out of the given region, with the given - // minimum and desired size. Returns the actual size allocated (between - // minimum and desired size) in actual_word_size if the allocation has been - // successful. - inline HeapWord* par_allocate(G1HeapRegion* alloc_region, - size_t min_word_size, - size_t desired_word_size, - size_t* actual_word_size); // Ensure that the region passed as a parameter has been filled up // so that no one else can allocate out of it any more. @@ -131,8 +116,7 @@ protected: static G1CollectedHeap* _g1h; virtual G1HeapRegion* allocate_new_region(size_t word_size) = 0; - virtual void retire_region(G1HeapRegion* alloc_region, - size_t allocated_bytes) = 0; + virtual void retire_region(G1HeapRegion* alloc_region) = 0; G1AllocRegion(const char* name, bool bot_updates, uint node_index); @@ -173,12 +157,6 @@ public: // Should be called before we start using this object. virtual void init(); - // This can be used to set the active region to a specific - // region. (Use Example: we try to retain the last old GC alloc - // region that we've used during a GC and we can use set() to - // re-instate it at the beginning of the next GC.) - void set(G1HeapRegion* alloc_region); - // Should be called when we want to release the active region which // is returned after it's been retired. virtual G1HeapRegion* release(); @@ -205,10 +183,9 @@ private: // in it and the free size in the currently retained region, if any. bool should_retain(G1HeapRegion* region); protected: - virtual G1HeapRegion* allocate_new_region(size_t word_size); - virtual void retire_region(G1HeapRegion* alloc_region, size_t allocated_bytes); - virtual size_t retire(bool fill_up); - + G1HeapRegion* allocate_new_region(size_t word_size) override; + void retire_region(G1HeapRegion* alloc_region) override; + size_t retire(bool fill_up) override; public: MutatorAllocRegion(uint node_index) : G1AllocRegion("Mutator Alloc Region", false /* bot_updates */, node_index), @@ -231,27 +208,36 @@ public: // This specialization of release() makes sure that the retained alloc // region is retired and set to null. - virtual G1HeapRegion* release(); + G1HeapRegion* release() override; - virtual void init(); + void init() override; }; // Common base class for allocation regions used during GC. class G1GCAllocRegion : public G1AllocRegion { + // When we set up a new active region we save its used bytes in this + // field so that, when we retire it, we can calculate how much space + // we allocated in it. + size_t _used_bytes_before; protected: G1EvacStats* _stats; G1HeapRegionAttr::region_type_t _purpose; - virtual G1HeapRegion* allocate_new_region(size_t word_size); - virtual void retire_region(G1HeapRegion* alloc_region, size_t allocated_bytes); + G1HeapRegion* allocate_new_region(size_t word_size) override; + void retire_region(G1HeapRegion* alloc_region) override; - virtual size_t retire(bool fill_up); + size_t retire(bool fill_up) override; G1GCAllocRegion(const char* name, bool bot_updates, G1EvacStats* stats, G1HeapRegionAttr::region_type_t purpose, uint node_index = G1NUMA::AnyNodeIndex) - : G1AllocRegion(name, bot_updates, node_index), _stats(stats), _purpose(purpose) { + : G1AllocRegion(name, bot_updates, node_index), _used_bytes_before(0), _stats(stats), _purpose(purpose) { assert(stats != nullptr, "Must pass non-null PLAB statistics"); } +public: + // This can be used to reuse a specific region. (Use Example: we try to retain the + // last old GC alloc region that we've used during a GC and we can use reuse() to + // re-instate it at the beginning of the next GC.) + void reuse(G1HeapRegion* alloc_region); }; class SurvivorGCAllocRegion : public G1GCAllocRegion { diff --git a/src/hotspot/share/gc/g1/g1AllocRegion.inline.hpp b/src/hotspot/share/gc/g1/g1AllocRegion.inline.hpp index ba4f1a12628..457a83f4285 100644 --- a/src/hotspot/share/gc/g1/g1AllocRegion.inline.hpp +++ b/src/hotspot/share/gc/g1/g1AllocRegion.inline.hpp @@ -31,9 +31,9 @@ #define assert_alloc_region(p, message) \ do { \ - assert((p), "[%s] %s c: %u r: " PTR_FORMAT " u: " SIZE_FORMAT, \ - _name, (message), _count, p2i(_alloc_region), \ - _used_bytes_before); \ + assert((p), "[%s] %s c: %u r: " PTR_FORMAT, \ + _name, (message), _count, p2i(_alloc_region) \ + ); \ } while (0) @@ -41,41 +41,27 @@ inline void G1AllocRegion::reset_alloc_region() { _alloc_region = _dummy_region; } -inline HeapWord* G1AllocRegion::allocate(G1HeapRegion* alloc_region, - size_t word_size) { - assert(alloc_region != nullptr, "pre-condition"); - - return alloc_region->allocate(word_size); -} - inline HeapWord* G1AllocRegion::par_allocate(G1HeapRegion* alloc_region, size_t word_size) { - size_t temp; - return par_allocate(alloc_region, word_size, word_size, &temp); -} - -inline HeapWord* G1AllocRegion::par_allocate(G1HeapRegion* alloc_region, - size_t min_word_size, - size_t desired_word_size, - size_t* actual_word_size) { assert(alloc_region != nullptr, "pre-condition"); assert(!alloc_region->is_empty(), "pre-condition"); - - return alloc_region->par_allocate(min_word_size, desired_word_size, actual_word_size); + size_t temp; + return alloc_region->par_allocate(word_size, word_size, &temp); } inline HeapWord* G1AllocRegion::attempt_allocation(size_t min_word_size, size_t desired_word_size, size_t* actual_word_size) { G1HeapRegion* alloc_region = _alloc_region; - assert_alloc_region(alloc_region != nullptr, "not initialized properly"); + assert_alloc_region(alloc_region != nullptr && !alloc_region->is_empty(), "not initialized properly"); + + HeapWord* result = alloc_region->par_allocate(min_word_size, desired_word_size, actual_word_size); - HeapWord* result = par_allocate(alloc_region, min_word_size, desired_word_size, actual_word_size); if (result != nullptr) { trace("alloc", min_word_size, desired_word_size, *actual_word_size, result); - return result; + } else { + trace("alloc failed", min_word_size, desired_word_size); } - trace("alloc failed", min_word_size, desired_word_size); - return nullptr; + return result; } inline HeapWord* G1AllocRegion::attempt_allocation_locked(size_t word_size) { @@ -112,7 +98,7 @@ inline HeapWord* MutatorAllocRegion::attempt_retained_allocation(size_t min_word size_t desired_word_size, size_t* actual_word_size) { if (_retained_alloc_region != nullptr) { - HeapWord* result = par_allocate(_retained_alloc_region, min_word_size, desired_word_size, actual_word_size); + HeapWord* result = _retained_alloc_region->par_allocate(min_word_size, desired_word_size, actual_word_size); if (result != nullptr) { trace("alloc retained", min_word_size, desired_word_size, *actual_word_size, result); return result; diff --git a/src/hotspot/share/gc/g1/g1Allocator.cpp b/src/hotspot/share/gc/g1/g1Allocator.cpp index 05c64287ec0..29ce8e26bbb 100644 --- a/src/hotspot/share/gc/g1/g1Allocator.cpp +++ b/src/hotspot/share/gc/g1/g1Allocator.cpp @@ -118,7 +118,7 @@ void G1Allocator::reuse_retained_old_region(G1EvacInfo* evacuation_info, // we allocate to in the region sets. We'll re-add it later, when // it's retired again. _g1h->old_set_remove(retained_region); - old->set(retained_region); + old->reuse(retained_region); G1HeapRegionPrinter::reuse(retained_region); evacuation_info->set_alloc_regions_used_before(retained_region->used()); } diff --git a/src/hotspot/share/gc/g1/g1HeapRegion.hpp b/src/hotspot/share/gc/g1/g1HeapRegion.hpp index dc2c71bcbb4..67d6556203c 100644 --- a/src/hotspot/share/gc/g1/g1HeapRegion.hpp +++ b/src/hotspot/share/gc/g1/g1HeapRegion.hpp @@ -127,18 +127,6 @@ private: void mangle_unused_area() PRODUCT_RETURN; - // Try to allocate at least min_word_size and up to desired_size from this region. - // Returns null if not possible, otherwise sets actual_word_size to the amount of - // space allocated. - // This version assumes that all allocation requests to this G1HeapRegion are properly - // synchronized. - inline HeapWord* allocate_impl(size_t min_word_size, size_t desired_word_size, size_t* actual_word_size); - // Try to allocate at least min_word_size and up to desired_size from this G1HeapRegion. - // Returns null if not possible, otherwise sets actual_word_size to the amount of - // space allocated. - // This version synchronizes with other calls to par_allocate_impl(). - inline HeapWord* par_allocate_impl(size_t min_word_size, size_t desired_word_size, size_t* actual_word_size); - inline HeapWord* advance_to_block_containing_addr(const void* addr, HeapWord* const pb, HeapWord* first_block) const; @@ -163,8 +151,18 @@ public: // All allocations are done without updating the BOT. The BOT // needs to be kept in sync for old generation regions and // this is done by explicit updates when crossing thresholds. + + // Try to allocate at least min_word_size and up to desired_size from this HeapRegion. + // Returns null if not possible, otherwise sets actual_word_size to the amount of + // space allocated. + // This version synchronizes with other calls to par_allocate(). inline HeapWord* par_allocate(size_t min_word_size, size_t desired_word_size, size_t* word_size); inline HeapWord* allocate(size_t word_size); + // Try to allocate at least min_word_size and up to desired_size from this region. + // Returns null if not possible, otherwise sets actual_word_size to the amount of + // space allocated. + // This version assumes that all allocation requests to this HeapRegion are properly + // synchronized. inline HeapWord* allocate(size_t min_word_size, size_t desired_word_size, size_t* actual_size); // Full GC support methods. diff --git a/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp b/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp index 8e5e594b5ca..f31c5fb553e 100644 --- a/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp +++ b/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp @@ -42,47 +42,6 @@ #include "utilities/align.hpp" #include "utilities/globalDefinitions.hpp" -inline HeapWord* G1HeapRegion::allocate_impl(size_t min_word_size, - size_t desired_word_size, - size_t* actual_size) { - HeapWord* obj = top(); - size_t available = pointer_delta(end(), obj); - size_t want_to_allocate = MIN2(available, desired_word_size); - if (want_to_allocate >= min_word_size) { - HeapWord* new_top = obj + want_to_allocate; - set_top(new_top); - assert(is_object_aligned(obj) && is_object_aligned(new_top), "checking alignment"); - *actual_size = want_to_allocate; - return obj; - } else { - return nullptr; - } -} - -inline HeapWord* G1HeapRegion::par_allocate_impl(size_t min_word_size, - size_t desired_word_size, - size_t* actual_size) { - do { - HeapWord* obj = top(); - size_t available = pointer_delta(end(), obj); - size_t want_to_allocate = MIN2(available, desired_word_size); - if (want_to_allocate >= min_word_size) { - HeapWord* new_top = obj + want_to_allocate; - HeapWord* result = Atomic::cmpxchg(&_top, obj, new_top); - // result can be one of two: - // the old top value: the exchange succeeded - // otherwise: the new value of the top is returned. - if (result == obj) { - assert(is_object_aligned(obj) && is_object_aligned(new_top), "checking alignment"); - *actual_size = want_to_allocate; - return obj; - } - } else { - return nullptr; - } - } while (true); -} - inline HeapWord* G1HeapRegion::block_start(const void* addr) const { return block_start(addr, parsable_bottom_acquire()); } @@ -225,9 +184,27 @@ inline void G1HeapRegion::apply_to_marked_objects(G1CMBitMap* bitmap, ApplyToMar } inline HeapWord* G1HeapRegion::par_allocate(size_t min_word_size, - size_t desired_word_size, - size_t* actual_word_size) { - return par_allocate_impl(min_word_size, desired_word_size, actual_word_size); + size_t desired_word_size, + size_t* actual_word_size) { + do { + HeapWord* obj = top(); + size_t available = pointer_delta(end(), obj); + size_t want_to_allocate = MIN2(available, desired_word_size); + if (want_to_allocate >= min_word_size) { + HeapWord* new_top = obj + want_to_allocate; + HeapWord* result = Atomic::cmpxchg(&_top, obj, new_top); + // result can be one of two: + // the old top value: the exchange succeeded + // otherwise: the new value of the top is returned. + if (result == obj) { + assert(is_object_aligned(obj) && is_object_aligned(new_top), "checking alignment"); + *actual_word_size = want_to_allocate; + return obj; + } + } else { + return nullptr; + } + } while (true); } inline HeapWord* G1HeapRegion::allocate(size_t word_size) { @@ -236,9 +213,20 @@ inline HeapWord* G1HeapRegion::allocate(size_t word_size) { } inline HeapWord* G1HeapRegion::allocate(size_t min_word_size, - size_t desired_word_size, - size_t* actual_word_size) { - return allocate_impl(min_word_size, desired_word_size, actual_word_size); + size_t desired_word_size, + size_t* actual_word_size) { + HeapWord* obj = top(); + size_t available = pointer_delta(end(), obj); + size_t want_to_allocate = MIN2(available, desired_word_size); + if (want_to_allocate >= min_word_size) { + HeapWord* new_top = obj + want_to_allocate; + set_top(new_top); + assert(is_object_aligned(obj) && is_object_aligned(new_top), "checking alignment"); + *actual_word_size = want_to_allocate; + return obj; + } else { + return nullptr; + } } inline void G1HeapRegion::update_bot() {