stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8209376: ZGC: Move ZMarkStackAllocator into a separate file

Browse Source 
Reviewed-by: eosterlund, kbarrett
This commit is contained in:
Per Lidén 2018-08-13 14:04:43 +02:00
parent f50c8dddfa
commit a51aeedca0
5 changed files with 257 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/hotspot/share/gc/z/zMark.hpp
View File

@ -25,6 +25,7 @@
#define SHARE_GC_Z_ZMARK_HPP
#include "gc/z/zMarkStack.hpp"
#include "gc/z/zMarkStackAllocator.hpp"
#include "gc/z/zMarkTerminate.hpp"
#include "oops/oopsHierarchy.hpp"
#include "utilities/globalDefinitions.hpp"

160
src/hotspot/share/gc/z/zMarkStack.cpp
View File

@ -22,169 +22,11 @@
*/
#include "precompiled.hpp"
#include "gc/z/zErrno.hpp"
#include "gc/z/zGlobals.hpp"
#include "gc/z/zLock.inline.hpp"
#include "gc/z/zMarkStack.inline.hpp"
#include "gc/z/zMarkStackAllocator.hpp"
#include "logging/log.hpp"
#include "runtime/atomic.hpp"
#include "runtime/os.hpp"
#include "utilities/debug.hpp"
uintptr_t ZMarkStackSpaceStart;
ZMarkStackSpace::ZMarkStackSpace() :
_expand_lock(),
_start(0),
_top(0),
_end(0) {
assert(ZMarkStackSpaceLimit >= ZMarkStackSpaceExpandSize, "ZMarkStackSpaceLimit too small");
// Reserve address space
const size_t size = ZMarkStackSpaceLimit;
const size_t alignment = (size_t)os::vm_allocation_granularity();
const uintptr_t addr = (uintptr_t)os::reserve_memory(size, NULL, alignment, mtGC);
if (addr == 0) {
log_error(gc, marking)("Failed to reserve address space for mark stacks");
return;
}
// Successfully initialized
_start = _top = _end = addr;
// Register mark stack space start
ZMarkStackSpaceStart = _start;
}
bool ZMarkStackSpace::is_initialized() const {
return _start != 0;
}
uintptr_t ZMarkStackSpace::alloc_space(size_t size) {
uintptr_t top = Atomic::load(&_top);
for (;;) {
const uintptr_t end = Atomic::load(&_end);
const uintptr_t new_top = top + size;
if (new_top > end) {
// Not enough space left
return 0;
}
const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, top);
if (prev_top == top) {
// Success
return top;
}
// Retry
top = prev_top;
}
}
uintptr_t ZMarkStackSpace::expand_and_alloc_space(size_t size) {
ZLocker locker(&_expand_lock);
// Retry allocation before expanding
uintptr_t addr = alloc_space(size);
if (addr != 0) {
return addr;
}
// Check expansion limit
const size_t expand_size = ZMarkStackSpaceExpandSize;
const size_t old_size = _end - _start;
const size_t new_size = old_size + expand_size;
if (new_size > ZMarkStackSpaceLimit) {
// Expansion limit reached. This is a fatal error since we
// currently can't recover from running out of mark stack space.
fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit=<size> to increase the "
"maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.",
ZMarkStackSpaceLimit / M);
}
log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M",
old_size / M, new_size / M);
// Expand
os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space");
// Increment top before end to make sure another
// thread can't steal out newly expanded space.
addr = Atomic::add(size, &_top) - size;
Atomic::add(expand_size, &_end);
return addr;
}
uintptr_t ZMarkStackSpace::alloc(size_t size) {
const uintptr_t addr = alloc_space(size);
if (addr != 0) {
return addr;
}
return expand_and_alloc_space(size);
}
ZMarkStackAllocator::ZMarkStackAllocator() :
_freelist(),
_space() {
guarantee(sizeof(ZMarkStack) == ZMarkStackSize, "Size mismatch");
guarantee(sizeof(ZMarkStackMagazine) <= ZMarkStackSize, "Size mismatch");
// Prime free list to avoid an immediate space
// expansion when marking starts.
if (_space.is_initialized()) {
prime_freelist();
}
}
bool ZMarkStackAllocator::is_initialized() const {
return _space.is_initialized();
}
void ZMarkStackAllocator::prime_freelist() {
for (size_t size = 0; size < ZMarkStackSpaceExpandSize; size += ZMarkStackMagazineSize) {
const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize);
ZMarkStackMagazine* const magazine = create_magazine_from_space(addr, ZMarkStackMagazineSize);
free_magazine(magazine);
}
}
ZMarkStackMagazine* ZMarkStackAllocator::create_magazine_from_space(uintptr_t addr, size_t size) {
assert(is_aligned(size, ZMarkStackSize), "Invalid size");
// Use first stack as magazine
ZMarkStackMagazine* const magazine = new ((void*)addr) ZMarkStackMagazine();
for (size_t i = ZMarkStackSize; i < size; i += ZMarkStackSize) {
ZMarkStack* const stack = new ((void*)(addr + i)) ZMarkStack();
const bool success = magazine->push(stack);
assert(success, "Magazine should never get full");
}
return magazine;
}
ZMarkStackMagazine* ZMarkStackAllocator::alloc_magazine() {
// Try allocating from the free list first
ZMarkStackMagazine* const magazine = _freelist.pop_atomic();
if (magazine != NULL) {
return magazine;
}
// Allocate new magazine
const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize);
if (addr == 0) {
return NULL;
}
return create_magazine_from_space(addr, ZMarkStackMagazineSize);
}
void ZMarkStackAllocator::free_magazine(ZMarkStackMagazine* magazine) {
_freelist.push_atomic(magazine);
}
ZMarkStripe::ZMarkStripe() :
_published(),
_overflowed() {}

41
src/hotspot/share/gc/z/zMarkStack.hpp
View File

@ -25,9 +25,7 @@
#define SHARE_GC_Z_ZMARKSTACK_HPP
#include "gc/z/zGlobals.hpp"
#include "gc/z/zLock.hpp"
#include "gc/z/zMarkStackEntry.hpp"
#include "memory/allocation.hpp"
#include "utilities/globalDefinitions.hpp"
template <typename T, size_t S>
@ -73,43 +71,6 @@ typedef ZStackList<ZMarkStack> ZMarkStackList;
typedef ZStack<ZMarkStack*, ZMarkStackMagazineSlots> ZMarkStackMagazine;
typedef ZStackList<ZMarkStackMagazine> ZMarkStackMagazineList;
class ZMarkStackSpace {
private:
ZLock _expand_lock;
uintptr_t _start;
volatile uintptr_t _top;
volatile uintptr_t _end;
void expand();
uintptr_t alloc_space(size_t size);
uintptr_t expand_and_alloc_space(size_t size);
public:
ZMarkStackSpace();
bool is_initialized() const;
uintptr_t alloc(size_t size);
};
class ZMarkStackAllocator {
private:
ZMarkStackMagazineList _freelist ATTRIBUTE_ALIGNED(ZCacheLineSize);
ZMarkStackSpace _space ATTRIBUTE_ALIGNED(ZCacheLineSize);
void prime_freelist();
ZMarkStackMagazine* create_magazine_from_space(uintptr_t addr, size_t size);
public:
ZMarkStackAllocator();
bool is_initialized() const;
ZMarkStackMagazine* alloc_magazine();
void free_magazine(ZMarkStackMagazine* magazine);
};
class ZMarkStripe {
private:
ZMarkStackList _published ATTRIBUTE_ALIGNED(ZCacheLineSize);
@ -145,6 +106,8 @@ public:
ZMarkStripe* stripe_for_addr(uintptr_t addr);
};
class ZMarkStackAllocator;
class ZMarkThreadLocalStacks {
private:
ZMarkStackMagazine* _magazine;

185
src/hotspot/share/gc/z/zMarkStackAllocator.cpp Normal file
View File

@ -0,0 +1,185 @@
/*
* Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
#include "precompiled.hpp"
#include "gc/z/zLock.inline.hpp"
#include "gc/z/zMarkStack.inline.hpp"
#include "gc/z/zMarkStackAllocator.hpp"
#include "logging/log.hpp"
#include "runtime/atomic.hpp"
#include "runtime/os.hpp"
#include "utilities/debug.hpp"
uintptr_t ZMarkStackSpaceStart;
ZMarkStackSpace::ZMarkStackSpace() :
_expand_lock(),
_start(0),
_top(0),
_end(0) {
assert(ZMarkStackSpaceLimit >= ZMarkStackSpaceExpandSize, "ZMarkStackSpaceLimit too small");
// Reserve address space
const size_t size = ZMarkStackSpaceLimit;
const size_t alignment = (size_t)os::vm_allocation_granularity();
const uintptr_t addr = (uintptr_t)os::reserve_memory(size, NULL, alignment, mtGC);
if (addr == 0) {
log_error(gc, marking)("Failed to reserve address space for mark stacks");
return;
}
// Successfully initialized
_start = _top = _end = addr;
// Register mark stack space start
ZMarkStackSpaceStart = _start;
}
bool ZMarkStackSpace::is_initialized() const {
return _start != 0;
}
uintptr_t ZMarkStackSpace::alloc_space(size_t size) {
uintptr_t top = Atomic::load(&_top);
for (;;) {
const uintptr_t end = Atomic::load(&_end);
const uintptr_t new_top = top + size;
if (new_top > end) {
// Not enough space left
return 0;
}
const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, top);
if (prev_top == top) {
// Success
return top;
}
// Retry
top = prev_top;
}
}
uintptr_t ZMarkStackSpace::expand_and_alloc_space(size_t size) {
ZLocker locker(&_expand_lock);
// Retry allocation before expanding
uintptr_t addr = alloc_space(size);
if (addr != 0) {
return addr;
}
// Check expansion limit
const size_t expand_size = ZMarkStackSpaceExpandSize;
const size_t old_size = _end - _start;
const size_t new_size = old_size + expand_size;
if (new_size > ZMarkStackSpaceLimit) {
// Expansion limit reached. This is a fatal error since we
// currently can't recover from running out of mark stack space.
fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit=<size> to increase the "
"maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.",
ZMarkStackSpaceLimit / M);
}
log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M",
old_size / M, new_size / M);
// Expand
os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space");
// Increment top before end to make sure another
// thread can't steal out newly expanded space.
addr = Atomic::add(size, &_top) - size;
Atomic::add(expand_size, &_end);
return addr;
}
uintptr_t ZMarkStackSpace::alloc(size_t size) {
const uintptr_t addr = alloc_space(size);
if (addr != 0) {
return addr;
}
return expand_and_alloc_space(size);
}
ZMarkStackAllocator::ZMarkStackAllocator() :
_freelist(),
_space() {
guarantee(sizeof(ZMarkStack) == ZMarkStackSize, "Size mismatch");
guarantee(sizeof(ZMarkStackMagazine) <= ZMarkStackSize, "Size mismatch");
// Prime free list to avoid an immediate space
// expansion when marking starts.
if (_space.is_initialized()) {
prime_freelist();
}
}
bool ZMarkStackAllocator::is_initialized() const {
return _space.is_initialized();
}
void ZMarkStackAllocator::prime_freelist() {
for (size_t size = 0; size < ZMarkStackSpaceExpandSize; size += ZMarkStackMagazineSize) {
const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize);
ZMarkStackMagazine* const magazine = create_magazine_from_space(addr, ZMarkStackMagazineSize);
free_magazine(magazine);
}
}
ZMarkStackMagazine* ZMarkStackAllocator::create_magazine_from_space(uintptr_t addr, size_t size) {
assert(is_aligned(size, ZMarkStackSize), "Invalid size");
// Use first stack as magazine
ZMarkStackMagazine* const magazine = new ((void*)addr) ZMarkStackMagazine();
for (size_t i = ZMarkStackSize; i < size; i += ZMarkStackSize) {
ZMarkStack* const stack = new ((void*)(addr + i)) ZMarkStack();
const bool success = magazine->push(stack);
assert(success, "Magazine should never get full");
}
return magazine;
}
ZMarkStackMagazine* ZMarkStackAllocator::alloc_magazine() {
// Try allocating from the free list first
ZMarkStackMagazine* const magazine = _freelist.pop_atomic();
if (magazine != NULL) {
return magazine;
}
// Allocate new magazine
const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize);
if (addr == 0) {
return NULL;
}
return create_magazine_from_space(addr, ZMarkStackMagazineSize);
}
void ZMarkStackAllocator::free_magazine(ZMarkStackMagazine* magazine) {
_freelist.push_atomic(magazine);
}

68
src/hotspot/share/gc/z/zMarkStackAllocator.hpp Normal file
View File

@ -0,0 +1,68 @@
/*
* Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
#ifndef SHARE_GC_Z_ZMARKSTACKALLOCATOR_HPP
#define SHARE_GC_Z_ZMARKSTACKALLOCATOR_HPP
#include "gc/z/zGlobals.hpp"
#include "gc/z/zLock.hpp"
#include "utilities/globalDefinitions.hpp"
class ZMarkStackSpace {
private:
ZLock _expand_lock;
uintptr_t _start;
volatile uintptr_t _top;
volatile uintptr_t _end;
void expand();
uintptr_t alloc_space(size_t size);
uintptr_t expand_and_alloc_space(size_t size);
public:
ZMarkStackSpace();
bool is_initialized() const;
uintptr_t alloc(size_t size);
};
class ZMarkStackAllocator {
private:
ZMarkStackMagazineList _freelist ATTRIBUTE_ALIGNED(ZCacheLineSize);
ZMarkStackSpace _space ATTRIBUTE_ALIGNED(ZCacheLineSize);
void prime_freelist();
ZMarkStackMagazine* create_magazine_from_space(uintptr_t addr, size_t size);
public:
ZMarkStackAllocator();
bool is_initialized() const;
ZMarkStackMagazine* alloc_magazine();
void free_magazine(ZMarkStackMagazine* magazine);
};
#endif // SHARE_GC_Z_ZMARKSTACKALLOCATOR_HPP