stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8255235: ZGC: Allocate and initialize forwarding data structures in parallel

Browse Source 
Reviewed-by: ayang, eosterlund
This commit is contained in:
Per Liden 2020-11-02 14:31:56 +00:00
parent 1019581ce2
commit 4c66b158a8
11 changed files with 172 additions and 153 deletions

6
src/hotspot/share/gc/z/zForwardingAllocator.inline.hpp

@ -25,6 +25,7 @@
#define SHARE_GC_Z_ZFORWARDINGALLOCATOR_INLINE_HPP
#include "gc/z/zForwardingAllocator.hpp"
#include "runtime/atomic.hpp"
#include "utilities/debug.hpp"
inline size_t ZForwardingAllocator::size() const {
@ -36,9 +37,8 @@ inline bool ZForwardingAllocator::is_full() const {
}
inline void* ZForwardingAllocator::alloc(size_t size) {
char* const addr = _top;
_top += size;
assert(_top <= _end, "Allocation should never fail");
char* const addr = Atomic::fetch_and_add(&_top, size);
assert(addr + size <= _end, "Allocation should never fail");
return addr;
}

48
src/hotspot/share/gc/z/zForwardingTable.cpp

@ -1,48 +0,0 @@
/*
* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
#include "precompiled.hpp"
#include "gc/z/zForwarding.inline.hpp"
#include "gc/z/zForwardingTable.inline.hpp"
#include "gc/z/zGlobals.hpp"
#include "gc/z/zGranuleMap.inline.hpp"
#include "utilities/debug.hpp"
ZForwardingTable::ZForwardingTable() :
_map(ZAddressOffsetMax) {}
void ZForwardingTable::insert(ZForwarding* forwarding) {
const uintptr_t offset = forwarding->start();
const size_t size = forwarding->size();
assert(_map.get(offset) == NULL, "Invalid entry");
_map.put(offset, size, forwarding);
}
void ZForwardingTable::remove(ZForwarding* forwarding) {
const uintptr_t offset = forwarding->start();
const size_t size = forwarding->size();
assert(_map.get(offset) == forwarding, "Invalid entry");
_map.put(offset, size, NULL);
}

24
src/hotspot/share/gc/z/zForwardingTable.inline.hpp

@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -25,12 +25,34 @@
#define SHARE_GC_Z_ZFORWARDINGTABLE_INLINE_HPP
#include "gc/z/zAddress.inline.hpp"
#include "gc/z/zForwarding.inline.hpp"
#include "gc/z/zForwardingTable.hpp"
#include "gc/z/zGlobals.hpp"
#include "gc/z/zGranuleMap.inline.hpp"
#include "utilities/debug.hpp"
inline ZForwardingTable::ZForwardingTable() :
_map(ZAddressOffsetMax) {}
inline ZForwarding* ZForwardingTable::get(uintptr_t addr) const {
assert(!ZAddress::is_null(addr), "Invalid address");
return _map.get(ZAddress::offset(addr));
}
inline void ZForwardingTable::insert(ZForwarding* forwarding) {
const uintptr_t offset = forwarding->start();
const size_t size = forwarding->size();
assert(_map.get(offset) == NULL, "Invalid entry");
_map.put(offset, size, forwarding);
}
inline void ZForwardingTable::remove(ZForwarding* forwarding) {
const uintptr_t offset = forwarding->start();
const size_t size = forwarding->size();
assert(_map.get(offset) == forwarding, "Invalid entry");
_map.put(offset, size, NULL);
}
#endif // SHARE_GC_Z_ZFORWARDINGTABLE_INLINE_HPP

9
src/hotspot/share/gc/z/zHeap.cpp

@ -65,7 +65,7 @@ ZHeap::ZHeap() :
_reference_processor(&_workers),
_weak_roots_processor(&_workers),
_relocate(&_workers),
_relocation_set(),
_relocation_set(&_workers),
_unload(&_workers),
_serviceability(min_capacity(), max_capacity()) {
// Install global heap instance
@ -377,8 +377,11 @@ void ZHeap::select_relocation_set() {
// Allow pages to be deleted
_page_allocator.disable_deferred_delete();
// Select pages to relocate
selector.select(&_relocation_set);
// Select relocation set
selector.select();
// Install relocation set
_relocation_set.install(&selector);
// Setup forwarding table
ZRelocationSetIterator rs_iter(&_relocation_set);

119
src/hotspot/share/gc/z/zRelocationSet.cpp

@ -22,51 +22,106 @@
*/
#include "precompiled.hpp"
#include "gc/z/zArray.inline.hpp"
#include "gc/z/zForwarding.inline.hpp"
#include "gc/z/zForwardingAllocator.inline.hpp"
#include "gc/z/zRelocationSet.hpp"
#include "gc/z/zRelocationSetSelector.inline.hpp"
#include "gc/z/zStat.hpp"
#include "memory/allocation.hpp"
#include "gc/z/zTask.hpp"
#include "gc/z/zWorkers.hpp"
#include "runtime/atomic.hpp"
#include "utilities/debug.hpp"
ZRelocationSet::ZRelocationSet() :
class ZRelocationSetInstallTask : public ZTask {
private:
ZForwardingAllocator* const _allocator;
ZForwarding** _forwardings;
const size_t _nforwardings;
ZArrayParallelIterator<ZPage*> _small_iter;
ZArrayParallelIterator<ZPage*> _medium_iter;
volatile size_t _small_next;
volatile size_t _medium_next;
void install(ZForwarding* forwarding, volatile size_t* next) {
const size_t index = Atomic::fetch_and_add(next, 1u);
assert(index < _nforwardings, "Invalid index");
_forwardings[index] = forwarding;
}
void install_small(ZForwarding* forwarding) {
install(forwarding, &_small_next);
}
void install_medium(ZForwarding* forwarding) {
install(forwarding, &_medium_next);
}
public:
ZRelocationSetInstallTask(ZForwardingAllocator* allocator, const ZRelocationSetSelector* selector) :
ZTask("ZRelocationSetInstallTask"),
_allocator(allocator),
_forwardings(NULL),
_nforwardings(selector->small()->length() + selector->medium()->length()),
_small_iter(selector->small()),
_medium_iter(selector->medium()),
_small_next(selector->medium()->length()),
_medium_next(0) {
// Reset the allocator to have room for the relocation
// set, all forwardings, and all forwarding entries.
const size_t relocation_set_size = _nforwardings * sizeof(ZForwarding*);
const size_t forwardings_size = _nforwardings * sizeof(ZForwarding);
const size_t forwarding_entries_size = selector->forwarding_entries() * sizeof(ZForwardingEntry);
_allocator->reset(relocation_set_size + forwardings_size + forwarding_entries_size);
// Allocate relocation set
_forwardings = new (_allocator->alloc(relocation_set_size)) ZForwarding*[_nforwardings];
}
~ZRelocationSetInstallTask() {
assert(_allocator->is_full(), "Should be full");
}
virtual void work() {
// Allocate and install forwardings for small pages
for (ZPage* page; _small_iter.next(&page);) {
ZForwarding* const forwarding = ZForwarding::alloc(_allocator, page);
install_small(forwarding);
}
// Allocate and install forwardings for medium pages
for (ZPage* page; _medium_iter.next(&page);) {
ZForwarding* const forwarding = ZForwarding::alloc(_allocator, page);
install_medium(forwarding);
}
}
ZForwarding** forwardings() const {
return _forwardings;
}
size_t nforwardings() const {
return _nforwardings;
}
};
ZRelocationSet::ZRelocationSet(ZWorkers* workers) :
_workers(workers),
_allocator(),
_forwardings(NULL),
_nforwardings(0) {}
void ZRelocationSet::populate(ZPage* const* small, size_t nsmall,
ZPage* const* medium, size_t nmedium,
size_t forwarding_entries) {
// Set relocation set length
_nforwardings = nsmall + nmedium;
void ZRelocationSet::install(const ZRelocationSetSelector* selector) {
// Install relocation set
ZRelocationSetInstallTask task(&_allocator, selector);
_workers->run_concurrent(&task);
// Initialize forwarding allocator to have room for the
// relocation set, all forwardings, and all forwarding entries.
const size_t relocation_set_size = _nforwardings * sizeof(ZForwarding*);
const size_t forwardings_size = _nforwardings * sizeof(ZForwarding);
const size_t forwarding_entries_size = forwarding_entries * sizeof(ZForwardingEntry);
_allocator.reset(relocation_set_size + forwardings_size + forwarding_entries_size);
// Allocate relocation set
_forwardings = new (_allocator.alloc(relocation_set_size)) ZForwarding*[_nforwardings];
// Populate relocation set array
size_t j = 0;
// Populate medium pages
for (size_t i = 0; i < nmedium; i++) {
_forwardings[j++] = ZForwarding::alloc(&_allocator, medium[i]);
}
// Populate small pages
for (size_t i = 0; i < nsmall; i++) {
_forwardings[j++] = ZForwarding::alloc(&_allocator, small[i]);
}
assert(_allocator.is_full(), "Should be full");
_forwardings = task.forwardings();
_nforwardings = task.nforwardings();
// Update statistics
ZStatRelocation::set_at_populate_relocation_set(_allocator.size());
ZStatRelocation::set_at_install_relocation_set(_allocator.size());
}
void ZRelocationSet::reset() {

11
src/hotspot/share/gc/z/zRelocationSet.hpp

@ -26,25 +26,24 @@
#include "gc/z/zArray.hpp"
#include "gc/z/zForwardingAllocator.hpp"
#include "memory/allocation.hpp"
class ZForwarding;
class ZPage;
class ZRelocationSetSelector;
class ZWorkers;
class ZRelocationSet {
template <bool> friend class ZRelocationSetIteratorImpl;
private:
ZWorkers* _workers;
ZForwardingAllocator _allocator;
ZForwarding** _forwardings;
size_t _nforwardings;
public:
ZRelocationSet();
ZRelocationSet(ZWorkers* workers);
void populate(ZPage* const* small, size_t nsmall,
ZPage* const* medium, size_t nmedium,
size_t forwarding_entries);
void install(const ZRelocationSetSelector* selector);
void reset();
};

65
src/hotspot/share/gc/z/zRelocationSetSelector.cpp

@ -25,7 +25,6 @@
#include "gc/z/zArray.inline.hpp"
#include "gc/z/zForwarding.inline.hpp"
#include "gc/z/zPage.inline.hpp"
#include "gc/z/zRelocationSet.hpp"
#include "gc/z/zRelocationSetSelector.inline.hpp"
#include "jfr/jfrEvents.hpp"
#include "logging/log.hpp"
@ -52,15 +51,9 @@ ZRelocationSetSelectorGroup::ZRelocationSetSelectorGroup(const char* name,
_object_size_limit(object_size_limit),
_fragmentation_limit(page_size * (ZFragmentationLimit / 100)),
_registered_pages(),
_sorted_pages(NULL),
_nselected(0),
_forwarding_entries(0),
_stats() {}
ZRelocationSetSelectorGroup::~ZRelocationSetSelectorGroup() {
FREE_C_HEAP_ARRAY(ZPage*, _sorted_pages);
}
void ZRelocationSetSelectorGroup::register_live_page(ZPage* page) {
const uint8_t type = page->type();
const size_t size = page->size();
@ -102,18 +95,11 @@ void ZRelocationSetSelectorGroup::semi_sort() {
const size_t npartitions = (size_t)1 << npartitions_shift;
const size_t partition_size = _page_size >> npartitions_shift;
const size_t partition_size_shift = exact_log2(partition_size);
const size_t npages = _registered_pages.length();
// Partition slots/fingers
size_t partitions[npartitions];
// Allocate destination array
assert(_sorted_pages == NULL, "Already initialized");
_sorted_pages = NEW_C_HEAP_ARRAY(ZPage*, npages, mtGC);
debug_only(memset(_sorted_pages, 0, npages * sizeof(ZPage*)));
int partitions[npartitions] = { /* zero initialize */ };
// Calculate partition slots
memset(partitions, 0, sizeof(partitions));
ZArrayIterator<ZPage*> iter1(&_registered_pages);
for (ZPage* page; iter1.next(&page);) {
const size_t index = page->live_bytes() >> partition_size_shift;
@ -121,39 +107,45 @@ void ZRelocationSetSelectorGroup::semi_sort() {
}
// Calculate partition fingers
size_t finger = 0;
int finger = 0;
for (size_t i = 0; i < npartitions; i++) {
const size_t slots = partitions[i];
const int slots = partitions[i];
partitions[i] = finger;
finger += slots;
}
// Allocate destination array
const int npages = _registered_pages.length();
ZArray<ZPage*> sorted_pages(npages, npages, NULL);
// Sort pages into partitions
ZArrayIterator<ZPage*> iter2(&_registered_pages);
for (ZPage* page; iter2.next(&page);) {
const size_t index = page->live_bytes() >> partition_size_shift;
const size_t finger = partitions[index]++;
assert(_sorted_pages[finger] == NULL, "Invalid finger");
_sorted_pages[finger] = page;
const int finger = partitions[index]++;
assert(sorted_pages.at(finger) == NULL, "Invalid finger");
sorted_pages.at_put(finger, page);
}
_registered_pages.swap(&sorted_pages);
}
void ZRelocationSetSelectorGroup::select_inner() {
// Calculate the number of pages to relocate by successively including pages in
// a candidate relocation set and calculate the maximum space requirement for
// their live objects.
const size_t npages = _registered_pages.length();
size_t selected_from = 0;
size_t selected_to = 0;
const int npages = _registered_pages.length();
int selected_from = 0;
int selected_to = 0;
size_t selected_forwarding_entries = 0;
size_t from_live_bytes = 0;
size_t from_forwarding_entries = 0;
semi_sort();
for (size_t from = 1; from <= npages; from++) {
for (int from = 1; from <= npages; from++) {
// Add page to the candidate relocation set
ZPage* const page = _sorted_pages[from - 1];
ZPage* const page = _registered_pages.at(from - 1);
from_live_bytes += page->live_bytes();
from_forwarding_entries += ZForwarding::nentries(page);
@ -161,14 +153,14 @@ void ZRelocationSetSelectorGroup::select_inner() {
// By subtracting the object size limit from the pages size we get the maximum
// number of pages that the relocation set is guaranteed to fit in, regardless
// of in which order the objects are relocated.
const size_t to = ceil((double)(from_live_bytes) / (double)(_page_size - _object_size_limit));
const int to = ceil((double)(from_live_bytes) / (double)(_page_size - _object_size_limit));
// Calculate the relative difference in reclaimable space compared to our
// currently selected final relocation set. If this number is larger than the
// acceptable fragmentation limit, then the current candidate relocation set
// becomes our new final relocation set.
const size_t diff_from = from - selected_from;
const size_t diff_to = to - selected_to;
const int diff_from = from - selected_from;
const int diff_to = to - selected_to;
const double diff_reclaimable = 100 - percent_of(diff_to, diff_from);
if (diff_reclaimable > ZFragmentationLimit) {
selected_from = from;
@ -176,24 +168,22 @@ void ZRelocationSetSelectorGroup::select_inner() {
selected_forwarding_entries = from_forwarding_entries;
}
log_trace(gc, reloc)("Candidate Relocation Set (%s Pages): " SIZE_FORMAT "->" SIZE_FORMAT ", "
log_trace(gc, reloc)("Candidate Relocation Set (%s Pages): %d->%d, "
"%.1f%% relative defragmentation, " SIZE_FORMAT " forwarding entries, %s",
_name, from, to, diff_reclaimable, from_forwarding_entries,
(selected_from == from) ? "Selected" : "Rejected");
}
// Finalize selection
_nselected = selected_from;
_registered_pages.trunc_to(selected_from);
_forwarding_entries = selected_forwarding_entries;
// Update statistics
_stats._compacting_from = selected_from * _page_size;
_stats._compacting_to = selected_to * _page_size;
log_trace(gc, reloc)("Relocation Set (%s Pages): " SIZE_FORMAT "->" SIZE_FORMAT ", "
SIZE_FORMAT " skipped, " SIZE_FORMAT " forwarding entries",
_name, selected_from, selected_to, npages - selected_from,
selected_forwarding_entries);
log_trace(gc, reloc)("Relocation Set (%s Pages): %d->%d, %d skipped, " SIZE_FORMAT " forwarding entries",
_name, selected_from, selected_to, npages - selected_from, selected_forwarding_entries);
}
void ZRelocationSetSelectorGroup::select() {
@ -241,7 +231,7 @@ void ZRelocationSetSelector::register_garbage_page(ZPage* page) {
}
}
void ZRelocationSetSelector::select(ZRelocationSet* relocation_set) {
void ZRelocationSetSelector::select() {
// Select pages to relocate. The resulting relocation set will be
// sorted such that medium pages comes first, followed by small
// pages. Pages within each page group will be semi-sorted by live
@ -255,11 +245,6 @@ void ZRelocationSetSelector::select(ZRelocationSet* relocation_set) {
_medium.select();
_small.select();
// Populate relocation set
relocation_set->populate(_small.selected(), _small.nselected(),
_medium.selected(), _medium.nselected(),
forwarding_entries());
// Send event
event.commit(total(), empty(), compacting_from(), compacting_to());
}

15
src/hotspot/share/gc/z/zRelocationSetSelector.hpp

@ -28,7 +28,6 @@
#include "memory/allocation.hpp"
class ZPage;
class ZRelocationSet;
class ZRelocationSetSelectorGroupStats {
friend class ZRelocationSetSelectorGroup;
@ -76,8 +75,6 @@ private:
const size_t _object_size_limit;
const size_t _fragmentation_limit;
ZArray<ZPage*> _registered_pages;
ZPage** _sorted_pages;
size_t _nselected;
size_t _forwarding_entries;
ZRelocationSetSelectorGroupStats _stats;
@ -91,14 +88,12 @@ public:
uint8_t page_type,
size_t page_size,
size_t object_size_limit);
~ZRelocationSetSelectorGroup();
void register_live_page(ZPage* page);
void register_garbage_page(ZPage* page);
void select();
ZPage* const* selected() const;
size_t nselected() const;
const ZArray<ZPage*>* selected() const;
size_t forwarding_entries() const;
const ZRelocationSetSelectorGroupStats& stats() const;
@ -110,7 +105,6 @@ private:
ZRelocationSetSelectorGroup _medium;
ZRelocationSetSelectorGroup _large;
size_t forwarding_entries() const;
size_t total() const;
size_t empty() const;
size_t compacting_from() const;
@ -121,7 +115,12 @@ public:
void register_live_page(ZPage* page);
void register_garbage_page(ZPage* page);
void select(ZRelocationSet* relocation_set);
void select();
const ZArray<ZPage*>* small() const;
const ZArray<ZPage*>* medium() const;
size_t forwarding_entries() const;
ZRelocationSetSelectorStats stats() const;
};

24
src/hotspot/share/gc/z/zRelocationSetSelector.inline.hpp

@ -66,12 +66,8 @@ inline const ZRelocationSetSelectorGroupStats& ZRelocationSetSelectorStats::larg
return _large;
}
inline ZPage* const* ZRelocationSetSelectorGroup::selected() const {
return _sorted_pages;
}
inline size_t ZRelocationSetSelectorGroup::nselected() const {
return _nselected;
inline const ZArray<ZPage*>* ZRelocationSetSelectorGroup::selected() const {
return &_registered_pages;
}
inline size_t ZRelocationSetSelectorGroup::forwarding_entries() const {
@ -82,10 +78,6 @@ inline const ZRelocationSetSelectorGroupStats& ZRelocationSetSelectorGroup::stat
return _stats;
}
inline size_t ZRelocationSetSelector::forwarding_entries() const {
return _small.forwarding_entries() + _medium.forwarding_entries();
}
inline size_t ZRelocationSetSelector::total() const {
return _small.stats().total() + _medium.stats().total() + _large.stats().total();
}
@ -102,4 +94,16 @@ inline size_t ZRelocationSetSelector::compacting_to() const {
return _small.stats().compacting_to() + _medium.stats().compacting_to() + _large.stats().compacting_to();
}
inline const ZArray<ZPage*>* ZRelocationSetSelector::small() const {
return _small.selected();
}
inline const ZArray<ZPage*>* ZRelocationSetSelector::medium() const {
return _medium.selected();
}
inline size_t ZRelocationSetSelector::forwarding_entries() const {
return _small.forwarding_entries() + _medium.forwarding_entries();
}
#endif // SHARE_GC_Z_ZRELOCATIONSETSELECTOR_INLINE_HPP

2
src/hotspot/share/gc/z/zStat.cpp

@ -1147,7 +1147,7 @@ void ZStatRelocation::set_at_select_relocation_set(const ZRelocationSetSelectorS
_stats = stats;
}
void ZStatRelocation::set_at_populate_relocation_set(size_t forwarding_usage) {
void ZStatRelocation::set_at_install_relocation_set(size_t forwarding_usage) {
_forwarding_usage = forwarding_usage;
}

2
src/hotspot/share/gc/z/zStat.hpp

@ -430,7 +430,7 @@ private:
public:
static void set_at_select_relocation_set(const ZRelocationSetSelectorStats& stats);
static void set_at_populate_relocation_set(size_t forwarding_usage);
static void set_at_install_relocation_set(size_t forwarding_usage);
static void set_at_relocate_end(bool success);
static void print();